{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "dcad5835",
   "metadata": {},
   "source": [
    "## SETUP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc983bba",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install gym[classic_control]\n",
    "!pip install dezero"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "758a1577",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# utility functions (common functions)\n",
    "def plot_total_reward(reward_history):\n",
    "    plt.xlabel('Episode')\n",
    "    plt.ylabel('Total Reward')\n",
    "    plt.plot(range(len(reward_history)), reward_history)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab0de828",
   "metadata": {},
   "source": [
    "## ch09/simple_pg.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b311d659",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode :0, total reward : 11.0\n",
      "episode :100, total reward : 26.0\n",
      "episode :200, total reward : 14.0\n",
      "episode :300, total reward : 22.0\n",
      "episode :400, total reward : 13.0\n",
      "episode :500, total reward : 13.0\n",
      "episode :600, total reward : 75.0\n",
      "episode :700, total reward : 57.0\n",
      "episode :800, total reward : 30.0\n",
      "episode :900, total reward : 82.0\n",
      "episode :1000, total reward : 74.0\n",
      "episode :1100, total reward : 176.0\n",
      "episode :1200, total reward : 74.0\n",
      "episode :1300, total reward : 41.0\n",
      "episode :1400, total reward : 76.0\n",
      "episode :1500, total reward : 79.0\n",
      "episode :1600, total reward : 94.0\n",
      "episode :1700, total reward : 187.0\n",
      "episode :1800, total reward : 28.0\n",
      "episode :1900, total reward : 23.0\n",
      "episode :2000, total reward : 133.0\n",
      "episode :2100, total reward : 69.0\n",
      "episode :2200, total reward : 52.0\n",
      "episode :2300, total reward : 113.0\n",
      "episode :2400, total reward : 47.0\n",
      "episode :2500, total reward : 101.0\n",
      "episode :2600, total reward : 83.0\n",
      "episode :2700, total reward : 63.0\n",
      "episode :2800, total reward : 40.0\n",
      "episode :2900, total reward : 102.0\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA/j0lEQVR4nO2deXwV1dnHf08SCMi+hEUWWUTZRMCIKygCbrTFpa9i3apWaqutuy9qa3GrtG/Vaq0LVqq4oUXrBqIIKCBrgBB2CBCWEJKwZIHsyfP+MTM3c29m7p25d+bOXZ7v55NP5p6ZOec5d+ae55znPOc5xMwQBEEQBABI8VoAQRAEIXYQpSAIgiD4EKUgCIIg+BClIAiCIPgQpSAIgiD4SPNagEjo3Lkz9+nTx2sxBEEQ4oq1a9ceZuYMo3NxrRT69OmDrKwsr8UQBEGIK4hor9k5MR8JgiAIPkQpCIIgCD5EKQiCIAg+RCkIgiAIPkQpCIIgCD5cUwpE1IuIFhPRFiLaTET3qukdiWgBEe1U/3dQ04mIXiaiXCLKIaKRbskmCIIgGOPmSKEOwIPMPBjAuQDuJqLBAKYCWMjMAwAsVD8DwBUABqh/UwC85qJsgiAIggGuKQVmLmDmdepxOYCtAHoAmATgHfWydwBcpR5PAjCLFVYCaE9E3d2ST3CeA8cq8P32Isfz/WFHMfYfrQjrXmbGx1n7UVPXEJEMlTX1eGvZHszNKQAAbDtUhrV7j0aUp5ss23kYeYdPRJxPfkklFm8zfqa5RcexYteRoPfX1DXg46z9WL/vGDbll/qd+zw7H2VVtX5pa/KOYvuh8pBybTtUhs/W5+OHHcVoaGBM+2Iz1u49inX7jmHLwTLfdQ0NyvM/dqIGX2w4GDTP4vJqzN90qMm9NXUNWLStEAdLKlHfwPh4zX7U1Td9n1bvOYqdheXYcrAMby3bg40HSvFx1n40NDBW7zmKHYXlKCyrwndbCpvUeYd639q9x/DCgh1+1zQ0MB79dCNyDpQAAPYfded3phGVxWtE1AfACACrAHRl5gL11CEAXdXjHgD26247oKYV6NJARFOgjCTQu3dv94QWbHPpi0tQUVOPvOkTHc331pmrkZZCyP3zlbbv/TKnAI/MycGBY5V4YMJpYcvw7LwteG/lPgDA8N6X4PK/LwUAx+vqFDe9tQpA5PJNfHkpSipqDfMZ/8IPIct4ZdFOvLwo1/dZu3b7oXLcOzsblw/phtdvPst3/n9eX2FJbu37B4Dp15yBt5fn4e3leU3K+c/a/fjfTzb60gd1a4MBXdsY5nnrzNXYUlCGzU9ehlbpafgy5yAemZOD/GOVeGnhTnRu3RwPTDgdj/13I0ora3HnmH5+91/3xgrDfBdvK8LXqrLp2aElDhyr9KufVudAtGu+zDmID1fvw4er9+GlycPxyJwcVNc1uPbuuT7RTEStAXwC4D5mLtOfY2WHH1u7/DDzDGbOZObMjAzDVdqCR1TU1LuWd11DeJtBlVYqPdEjx6sjKr+orPH+SEcd8URJRW3oi4JQfLzGML2ipg4AUFBWFVH+AHAsiIyB8lfVmj87bTRar248pr07R08odTh8vAbHKpTjoxXG9TLia93o48CxSsv3aZRVNtbh3tnZqHb5/XNVKRBRMygK4X1m/lRNLtTMQup/bRyUD6CX7vaeapogeI7sT5g8BG5GyQZPP5zGPVyIKGplAe56HxGAtwBsZeYXdKe+AHCrenwrgM916beoXkjnAijVmZkEISKkUU9OAp+7UQPvQ217V+4+gtr6Bhg1xVr7/GWI+QknibJOcHVO4QIANwPYSETZatpjAKYD+JiI7gCwF8B16rl5AK4EkAugAsBtLsomJAlO/Z5kK/NwMf7iYvnr/PW7a/HrMf3Qs0NL18pgZhQfr0ZFtbnJtbSyFu1aNgM59hZbwzWlwMzLYP6bHGdwPQO42y15hOQklhsfwTmlbUagMt9ysAzDerYPKUtu0XGfUnCrQzDq2YVBz1/3+gp8c/+YqI8UZEWzkBRE+XclWCTaSnvqpxuxu/h4lEttihVFs71Qcc2N9rsrSkFICmTEEFuEauiKyqpworoOZVW1OByh51ggmjdRIGVVdY6W4xSJNKcgCJ4jI4T4ZNSfF6Jf51Y4VFblytoXS7jYGh+vsa6Aoj2nICMFQbCEjDWsUt/AIdeFWPk2dx8+4eraF6tYkXXRtsLQF+k4/7lF1i+WOQVBcB7xHooeT3+1BWc9812TEBZGeDWSK6uqRX2IBZGGLqkmEutDa1jheLWdkUJ0EaUgCDYRk1RwvtmsrOA9bsFG77auNlqXcLy6DsOmfYvn5m0Nca+1/AAgLdW9pjRhFq8JQizh5O/Ki0FHQwOH7NnGIkYjtPoG9nTkpvXSv8xxbgFaWop7DXe0OyEy0SwIccCVLy/FtkPlMRuAT0+oBr//Y/OiI4gJVidu7ZiP3FQKKVHuustIQUgKIu2Zej0nsc1COOlYw8rozImm1O4oMGioC6PrLVzupvno/o82uJa3EaIUhIQm2j7eTlFUVoU+U+fi82yJCRmKcBV2qBGDnXcnXt8zI0QpCAmNUz38aA8UdhYpq24/WrM/xJWC4CyiFISkwMmeXAJ1ChMep81+iTQiMEMmmoWkwMnGIf58gBIbs4Z6+FPfGm6IFO0VwvGGKAUhoUmGnl2sYXci1y3Mdo2zL19s1CdaiPlIEARXIFBUvLbcKkMbUXjteRZtRCkIggU4DlqGfy7OxW/fXxv1ct9atge3v70m6uVGitVRpP7RT/96mzvCmHDv7PVRLQ8Q85Eg2CZWLVL/9812T8p9+qstnpTrNprS8NIc9nl29Lb91HBzj+aZRFRERJt0aR8RUbb6l6dt00lEfYioUnfudbfkEsyZuWwPnvs6eDwYQYg13Jo3ioPBoSu4OVJ4G8ArAGZpCcx8vXZMRM8DKNVdv4uZh7sojxCCp9Qe36NXDPJYkuAs2FKIH3MPY9rPhkStTDY5Tkb+On8bTu/WxvR8sjamiYKbezQvIaI+RudICft3HYBL3CpfSFzunJUFADaVQvK0VFM/yXE1/1e/32XpOqLY8UQKB5/5KH6rEBZeTTSPBlDIzDt1aX2JaD0R/UBEo81uJKIpRJRFRFnFxcXuSyrENeH4pO85fALTvtiMhjiMSgoAs2UVtCWsvhlW3oJEWvvglVK4AcCHus8FAHoz8wgADwD4gIjaGt3IzDOYOZOZMzMyMqIgqhDPhNNT/fW7WXh7eZ4v1EQgifPzdwc737gX60is9vw10coqQ28WFM8jokCirhSIKA3ANQA+0tKYuZqZj6jHawHsAnBatGUTBKCx0dA3WLFmQjhwrAIvfLs9Llxl451vt9jbajPe8WKkMB7ANmY+oCUQUQYRparH/QAMALDbA9mEBCOSYX0st7d3vbcWLy/KRa7JaMZL7HzjXnzHTo1Ofsw93JhnAo0f3XRJ/RDACgCnE9EBIrpDPTUZ/qYjABgDIEd1UZ0D4C5mPuqWbEJicLCkEm8t2+N4vqEajVjQFdW1SkyfWJAlkFiUyQ1u/Ncq33EimY/c9D66wST9lwZpnwD4xC1ZhMTk9rfXYNuhclwxtBtObt/SsXxjeYQQTxC8+y6DmdXk+QZHwlwIcUu5ujF8Q5R/5YljKPAetyaaf9gR2jORQhRua5OdBHorRCkIQgBGjUEsdS6X7zps6hkVC9jR0ev3lbgyL1JtEDI7kPySSmwtKIu4rLV7j2HD/pKI84kVRCkIcU+ymQN+8eaq0BfFAhY7z+Nf+CHkNQdLKrF4e1GEAjXlipeWAgDW7zvW5JzV3v+1ry3HR1mJszZElIIgWEBcP8PAwa9s4stLcdu/rUdiDdacG40Er351eZO0RJo8toMoBcET1u87hoMllVErT3Ze8+dQaRXW7lV6xzV1DVhg4ItfVFYVbbFMOWayYU44iH4PjigFwROufnU5zp++yPVyZOc1Yy7+22Jc+5rSO/7bt9tx56wsrNh1xO+aK19eFlkhcf7dJ9LksR1EKQgJTSS9QjPzQTw1FZvyS1Fq0Muuqm2ciN13pAIAUFJR43fN4ePVEZW9u/gECkq9GW2E8iwSzJFNdgQhgETqIf7kH8swsFsbzL9vjOk1ztvOlfwmz1jpcL7WceIJ7j9W4UAu7lFaUYt2JzVzPF8ZKQgJTTgdxkSbYNx2qNzSddK59ucfi3K9FiEoV7/2oyv5ilIQBCGu2FFoTcklOruLT7iSrygFQQggkcxHicaPuYdx6YtLvBYjoRGlIAgWSAY3xoMlseOCasbu4thdyZ0oiFIQ4hYnbOAlFTWorQ8dEsGr/KKBpvCe+moLcg6UeCpLKGrqrWnnYO9GEuj3iBClICQ1w59agAc/3uBIXg0NjOFPLcAjc9zdI9lNnLBTuzmqevqrLY7nWWphZ7VkQpSCkBQEa6i+2HAw7Hwra+oby1D/f56dH3Z+RtTH6V7R0cKuEgocRJz55LeOyZIIiFIQEhrLm7MbtCz6JDM31UFPzMeaPGf3g6oLMD/98t+rHc0/0Ug0F2KvEaUgJC16RTBjSePur3bnKrLy/CNsRtJE5R0+gVMf/xqfrW8cbSzdeTjIHYIRMqcQPm5uxzmTiIqIaJMubRoR5RNRtvp3pe7co0SUS0Tbiegyt+QSEodCNWBbMPOB1QZgzlrfluH2zREmDdAri3aiz9S5qKqtN77AAG2h2byNBfaEiABpJAU9bo4U3gZwuUH6i8w8XP2bBwBENBjK3s1D1HteJaJUF2UTEoBai54o0UbTEW8vzwPQuEOcpXtjdInE/32zDX2mzg16TZ+pc13Z88Bp3vhhl9cixDSuKQVmXgLAqrF1EoDZzFzNzHsA5AIY5ZZsQvIQThtrt2EOvDxQVYVj846Wurtv9npL1/1zsbWG9MsNB2Ni5JGeZt6n3Hww8t3WNFbtPhL6ojjDizmFe4goRzUvdVDTegDQb110QE1rAhFNIaIsIsoqLg69D6sgmGHHTBStxWvRHih8lh2+51WsYPRsOrVuHpWyI/Fci1WirRReA9AfwHAABQCet5sBM89g5kxmzszIyHBYPMEtjhyvRp+pc7HEwobqN8xYiXs+WOdo+U56qJiNJGJ5d7Y+U+dGHArbErH7FQgWiapSYOZCZq5n5gYAb6LRRJQPoJfu0p5qmpAg5OSXAgDeWrYn5LUrdh/BVznRm2i1gp97akDDF6gkYnRawHS1cjR1WZbD7rsA8PWmpu9KDOvnmCeqSoGIuus+Xg1A80z6AsBkIkonor4ABgAQ52zBElZGAUZB7sJtN+77KDuEPI2lJh0hqqx3/XWKTfnOzRHYJVYdAyLBtU12iOhDABcD6ExEBwD8CcDFRDQcyu8mD8CvAYCZNxPRxwC2AKgDcDczW/fjEwQXMOttZu8v8fscMqpqGNonnnu6sWxGE0LjmlJg5hsMkt8Kcv2zAJ51Sx4hcbES6jpeVr1q20jakfeX/16N6zN7hb7QIou3F4XvthkfX7MQBFnRLEQVt9uMorIqXPvact+kqtXhfSRmAO1e07qFkbd+D+VQfL+9GL9537mJ+d++tw4rdztv+48mMlgJH1EKQtyj71X/e3ke1u49ho/WKB7OQVc7606GakQiGmkY3NrQwLhv9nqs36eEyHhu3lbM31Tg0SyE9br9Zf624BeEqEAi2uATDVEKQkLy5lL/CU0vdlML1gAeOVGDz7IP4s5ZWQCAN5bsxl3vrUOFjZAYgDtT2cHkfu37EGalCHTnN5sPhX+zRyTiLn2iFISEQuvxl1T4x8iP5pyC2ahj35EKPPH5piahsPX2e30MJktl2ZbOXT5dH74n+a/fXeuYHJFuFlSbxOHKRSkIccOsFXm273EqWmYw85JvcjhEhv/7SQ5mrdiLtXv9o6o+93WjScZNz51o9WqD1SBaMkz9dGNE9y/dmbzREkQpCFHBiabgic83G6ZvO1SONyP0f49oojlEXloj2SxN+bmdqGkMkBfNCVEro6UPVu9DRY14gyczrrmkCoIeN9s+zexw55h+5uWHmkh2UEAtr0Bl0TxVUQo1dbG1h/N3Wxsjm67e443XkaxtiB1kpCAkLNV1kfV4rc5DaCODFQERM63cHak3TqJMc67bdyz0RTFIInpTiVIQokK0fjv6hlwfZ8nox2srSqqFa26daS0yi5PfhZ3+dSx7ytTH1uApqRGlICQsdhaAGSmNSBtRK3cnm9XErGcdryOFnYXHvRbBcUQpCAnFj7nG+xmH0/g65cYaKw1/tNxyw6nv9K9DLIqLMlbrEGgyTAREKQhRxc6EYkMYvuL6iJmE4L19fSOpF4tsGoqNrl6TdxRF5YH7FzStT7Rt0gu2FKK4iVyC0Iip9xERPRDsRmZ+wXlxBKGRz7Lzcc3InhHl4VUgvP95fYXpOb3SiXQUYUen1NQ14K731mFgtzaRFRqC8qpa03OJODGbaARzSdXenNMBnA1lzwMA+ClkrwMhCgSuSnaKH3MPY2iPdmjRrHGgvP9YRfCbLCxes4P57m22s7KMNpm790iIukZIEi8GTghMzUfM/CQzPwllF7SRzPwgMz8I4CwAvaMloJC8RNq2EDU1H5VW1uLGf63CXQEhFYwmpVfscs9e7FTjb2tVdswFxRBiEStzCl0B1Og+16hpgmCbcHrVTqItHNtZVB6yYX5m7lbsPXICQPAGNVSVtHuNyvPi6xATjhAMKyuaZwFYTUT/VT9fBeDtUDcR0UwAPwFQxMxD1bT/g2J+qgGwC8BtzFxCRH0AbAWwXb19JTPfZb0aQrwQbytXy6vqQl4TSRsbZ1+HkAQEHSmQ0q2bBeA2AMfUv9uY+TkLeb8N4PKAtAUAhjLzMAA7ADyqO7eLmYerf6IQBEcVSKQ5HXN4fsNqb/1QaVXwfByQRRD0BB0pMDMT0TxmPgOAra2dmHmJOgLQp32r+7gSwM/t5CkIdjByR62xuXRWa7xzi4IsUgrDhdWqvtt9+IStvIMRC6OSWF5VLShYmVNYR0Rnu1D27QC+1n3uS0TriegHIhptdhMRTSGiLCLKKi5O3vC28YYXcwlVuthHWulTP8kBAJRZMAsp90Uut9YYG7XJR07UGKTayDuMa6VZFoJhRSmcA2AFEe0iohwi2khEOZEUSkSPA6gD8L6aVACgNzOPAPAAgA+IqK3Rvcw8g5kzmTkzIyMjEjGEOMSOSclolzBtxXOwSKV2G81Q1xcYmIAaTOphlq5x5Hh8Lzw7VBbcHCZYp2eHlq7ka2Wi+TInCySiX0KZgB7H6i+cmasBVKvHa4loF4DTAGQ5WbbgDXX1DSipsNYjDtyAximMmtpomVOufW05Xr/pLFTpttr83Qfrw8rrrGe+C1uOWBghuPV8k5E+nVq5km9IpcDMewGAiLoAaBFJYUR0OYBHAFzEzBW69AwAR5m5noj6ARgAILJdU4SYYdqXm/Heyn2+z0VBeos7CsudLVzXEpZV1SpmLA+M69n7S7BcXfewMb8Uq/OM9y2wK5qdhr46xvZxECLDLYtsSPMREf2MiHYC2APgBwB58J8LMLvvQwArAJxORAeI6A4Ar0BZKb2AiLKJ6HX18jEAcogoG8AcAHcxsze7fQiO81VOgd/nwjJrJhCjBtJ2e65ev6OwHMOmfdtkf+RQ1DU0oM7huM4vLdzpaH5Weeg/GwC4P78j6yDiGyvmo6cBnAvgO2YeQURjAdwU6iZmvsEg+S2Taz8B8IkFWQTBlGDzDdsOhTcC+dkrPyKjTXrQa6w0glZXE9tddRzOmMfttSKx4OUkhI+VieZaZj4CIIWIUph5MYBMl+USEghPO45a4UajDotNaqioopY8lGKooTwhezALQbAyUighotYAlgB4n4iKADjnPC0IOkI1rzHUtrpCIvSyPZq2STrcMgNaGSlMAlAB4H4A86GEp/ipK9IIQhxizXxkDWlLBau4NQK3MlKYDGAJM+8E8I5LcghJhFsTkcF6p0amokToza4x8WLykkT4XpMZK0qhN4A3iKgvlHUDSwAsZeZsNwUTEgc7w1z9pV6FerartKztxWxxotlmi/rVhoLQFwmCDUKaj5j5T8x8CYDBAJYCeBjA2uB3CYI79H9sXlj3eR1zx63ec6gV0ELikuLSKx1ypEBEfwBwAYDWANYDeAiKchCEuMHKqKOkogajnl0YdtA8J5A2XvAaK+aja6DEKZoLZfHaCjUshSAAAJ6btxUHjlWanj+qC/q2dOdhANuiIJWCzyPV0CXVn/X7S2wrBKu41dbLQjHBaayEuRipBqe7AMAEADOIqIiZL3RdOiEueGOJvYgkimJQKCqrQpe2jdFT9GYeZuBgibmyCcSrTraTpqksm7GBZGSRvHjmkkpEQwHcCOBWANcDyAewyBVphKRj3sbgE6WfrjsQUf6x0ma61XjHSv2E6OOlS+p0KHMILwNYw8zObkElJBzM7PlezFZwLNyDg2EubCNaIWlx6ydmxXz0EyJqCWW/A1EIgqM0adPI/1yk7XaQKBeCIBhgxXz0UwDZUFYzg4iGE9EXLsslxDi5ReUY+qdv8O6KPNfKqK1rwPMLdli+3u1Ab2Y8MicnZNli+xfiBSthLqYBGAWgBADURWt9XZNIiAumzFqL49V1+OPnmyPKJ9gIePH2Ilt5nfYH84jubiuMw8cj21ZTEOzjXeyjWmYuDUiTfo/gCE6+SDa3SmhSdiQ/Ma9WXwvJi2eb7ADYTES/AJBKRAOI6B8AlrsjjhA3OPRC/mNRLvYfrTA852QzGw8T3+EgykhwGitK4XcAhkDZQ/lDAKUA7nVTKCH2caqJPXqiBne8swazV+/D5oP+A1IzZWGHRA8DseewRLFPVtzq5liJfVTBzI8z89nMnAngXSjbaoaEiGYSURERbdKldSSiBUS0U/3fQU0nInqZiHKJKIeIRoZbKcFb7LbDVbUNmPrpRkx8eZnfi+6EnX7RtiJVptBRUn/z3rqIyzPDrTkNmcsQnMZUKRDRMCL6log2EdEzRNSdiD4BsBDAFov5vw3g8oC0qQAWMvMANa+pavoVAAaof1MAvGa9GoLGviMV+GRtZAu+rBAv5pi6eqUxttIkV9bKjmRC/ODFnMKbAD4AcC2Aw1DcUncBOJWZX7SSOTMvARAY8H0SGvdleAfAVbr0WaywEkB7IupupRyhkate/REPqhu06/lsfT4OlVaZ3vd5dr6bYgXFTbt4MA+m7216N0VCYhuxBC9wK/JvMKWQzsxvM/N2Zv47gBPM/Agzm7cs1ujKzFpsg0MAuqrHPQDs1113QE3zg4imEFEWEWUVFxdHKErisGhbIfYfrfALPqdRUVOH+z7Kxi/eXGl4b119A+6dne2YLN/vKMLeI+HZusur6hyTAwjukeRknUOR4FMbQgIRbEVzCyIagcb5jGr9Z2aO2ADLzExEtn4uzDwDwAwAyMzMlJ+ayu1vZ6F5qrGO1xrGwjJjfR7Olxisj3L721kAgLzpE23n+9RXVi2T8cX2wnKvRRASDC/CXBQAeEH3+ZDuMwO4JMwyC4moOzMXqOYhbQyfD6CX7rqeappgEbfCPgeycvcRFJVbi55uJ8ppIlNZI/MVgrNEXSkw81h3isQXUCKuTlf/f65Lv4eIZgM4B0CpzswkeEBtfQOW7CjGuEFd/dInzzA2QxlRXRdaUe0/6r7i8Np8I+sJhHjByjqFsCGiDwGsAHA6ER0gojugKIMJRLQTwHj1MwDMA7AbQC6USe7fuimb0IhZg/n373bgjneysHSnzN0IQqzh1kSzldDZYcPMN5icGmdwLQO42015kgknwlfvU3vwRpPXgj28HqkICYiHYS6EBMfMtOFV1NFExG5cJkHwCtORQqgVxU54HwmxgbT9giBoBDMfPR/kXCTeR0KcEKn5aXfx8YSPPWQVGXUJThP17Thd9D4SogBzo8ua3QYpv6QSXdqkN7mvpKLGlqK45PkfMH5QF1tlC4JgDbdCzViaaCaioQAGA2ihpTHzLFckEqJOoM64YPoi3HhOb99n7eUb/tQC23mv2hMY5SRBkYGAEGU8i5JKRH8C8A/1byyAvwL4mUvyCC4QTo/ihx2NbqiRmD7qZYYVQOKH8BYSByveRz+H4kJ6iJlvA3AmgHauSpWEVNXWo9bBFcn6Jijk/sEudnMrZCWvIMQVVpRCJTM3AKgjorZQwlL0CnGPYJOBf5yPn7/mzYZ2oTqx8RIm20tqQ4yIZKAgOI0XsY80soioPZRVxmsBHIeySllwmA0HArfCdoZwGnXRA/aoCrEXg5iPBKeJuveRBjNr4SZeJ6L5ANoyc45L8ggOoZiMrL02Rs2VW0voExVtMx8zdhXLtplCfGBlonmhdszMecyco08T4h/xoY8cmVAXoo1bZt1g23G2IKKOADoTUQd1b+WORNQHBpvfJBPHq+vQZ+pcvLsiz2tRALjXqEszZ526huiELRcEDS9cUn8NZQ5hIIB16vFaKKGuX3FJnrigWN1L4K1lezyWRCFSnRDqdjEkheZnr/zotQiC4AimSoGZX2LmvgAeYua+ur8zmTmplUI8cOrjX2O1xYVjRkpFPzKVEYMgxB5pqdHfo1njDSL6PRHNUf/uIaJmrkgTZ3jdWN4yczV+/+F6Uzlmr9kXdt4yOhCE2KbDSc1dydeKUngVwFnqf+34NVekiRNipcFcsqMYX2w4GPmcQqh1CpHlLgiCG0R7nQIRpTFzHYCzmflM3alFRLTBHXGEcAilEsJZ0Zx3pAJDesjCdUFINoKNFFar/+uJqL+WSET9AIQdu4CITieibN1fGRHdR0TTiChfl35luGUkG655lHptHxMEwRS31hIFUwpaiQ8BWExE3xPR9wAWAXgw3AKZeTszD2fm4VBMURUA/queflE7x8zzwi0jWjADq/ccxXVvrLAVt2jl7iOYPGMF6hyKdRRp7CJZpiAIgkawFc0ZRPSAevwGgFT1uB7ACACLHSh/HIBdzLw3XuPrPPSfDdh3tAIHSypxSqdWlu65/6NsFJRWoai8Gie3bxmxDKEade30iZp6lFfVok2LZobnmxCfj0QQkgK3msxgI4VUAK0BtIGiPEj9S1PTnGAygA91n+8hohwimklEHYxuIKIpRJRFRFnFxcVGl8QNrnfQDQr4ZnNhRPcLgpDYBBspFDDzU24VTETNoezL8Kia9BqAp6E0RU9D2Q709sD7mHkGgBkAkJmZ6UmzFamGdlrBR+x8FCKDOB3ECUJC48WKZrebgisArGPmQgBg5kJmrlfDdL8JYJTL5UdM5LZ8Z3SaqRy+7ThD3R/ivIwYBCFpCKYUxrlc9g3QmY6IqLvu3NUANrlcfthEOuvv9PzJ4fIaR/MTBCH2ifqcAjO7trkuEbUCMAHAp7rkvxLRRiLKgbLt5/1ulR8tGhoYf/tmO4rKq1wt5/HPNkZ0f+hNdiLKXhCEOMLKJjuOw8wnAHQKSLvZC1kigbmxwTRqWNfkHcUri3OxMb8U79weuTXsqS+3oK6hAU9NGuqXXl0XwrU1pPlI7EOCEG94sU5BiJB6VVNU1xmv9bNrq5/54x7MWrHXIB9p1AVBcAZRCjbYd6QC/11/wPBcNE0s76/yVwxm+7ss2VEMZsa/l4cI8S06RRDiDi/WKQgBTPrnMtz/0QbDhxGss+50R/7x/27CviMVvs9m+/8ePl6DlbuP4u/f7Qyan+gEIV65ekRS7/flCqIUbHCsotbvc8gJWgOb34nqOuSXVEYsS61up69gcgSarux0LmSuQYh1ktl06sU6BcEFpn2x2fE8g/0wrLi/hqPcBCEWSF6V4B6iFMLAqBHdc+SE+fW64yJ1K0+zfMJhw4FS03M7DpWHvN9sRKANRmTEIAgxiEuTCqIUwsCokbzt32ss3Rttn/9n520Nej6/pBKb8ssMz83ffMgNkQTBMZLYeuQaohTCQP8iWmnkzS6x2gMvr6rFPxb6TxbXhFqbYJELpi/CnbOygl7zY+5hR8oSBKf56Zkney2CZ7jVv/Rk8Vq8o2/KrfRU7HRmKmrqcFJz/8dy1jPfNVECT37p/NyEGR+u3o+xp3eJWnmCYJUJg7t6LULCISOFMLDq8WA0itAnHa+uw85Cf5t/bX3TvI1GBblFxy3J4BS/fX9dVMsTBCE45/XvFPqiMBCloLK1oAz1ZqvAAnDKjHnrzDWY8OKSsO6N9qZEdRa/G0EQosO5/UQpuMbmg6W44qWleGVRbtDrgsU5skJhWRWOV9f5Ph8+Xt30IpfbXgluJwhCMGROAUBBiRLFNOdAicU7GltuS42sevk5f15o4VKLpilLVxnkLx1+QRCCIErBBgSlfdcsKcrcgjdd75QIu/zlVbVRN0EJghD7iFKwAREBzKaxhsxvtH5pYNab8o0XplXU1Bmmh8wfwNETNRj59IKw7hcEIbGROQUbaG27LuyQoTmmqrYetfX6i8Iv08zLqKwqPKUwc9keUQiCIJgiSkFHqLbbN9Gsu7Kuvqm76MA/zsc1ry53xLDktIVnS4Hx6mVBEPwZ2qOt1yJ4gmdKgYjy1O03s4koS03rSEQLiGin+r+DF7Ixc9C1CPpTB0urdOmN9200MfuYlec7tiGnIAju8ZNhsbtaekCX1q7l7fVIYSwzD2fmTPXzVAALmXkAgIXq56jz3qp96PvoPBSX+7uMatFCzfTFc19vQ99H5zVJl4ByghB/TBndD7+6sK/XYhgy567zXcvba6UQyCQA76jH7wC4ygsh5qxVdlf7bH0++kyd28RVVZtoDmzqZyzZ7ffZqnePXskEjlDunZ1tKQ9BEJwlJYVwdt+OXovRhBWPXoJ2JzVzLX8vlQID+JaI1hLRFDWtKzMXqMeHADQJbEJEU4goi4iyiouLHRWIAPSZOhcb9pcAABZuKwQALN15uPECmO905gQyphAEIRgdTmruav5euqReyMz5RNQFwAIi2qY/ycxMRE3aSGaeAWAGAGRmZrrahga2/T7vI4ulWo2RlLX3mGmZgiB4R0MMhndxu43wbKTAzPnq/yIA/wUwCkAhEXUHAPV/kVfyAU177Y3WIGefymfZ+boyY+8lFIRkpT4Je2meKAUiakVEbbRjAJcC2ATgCwC3qpfdCuBzt2Q4WFKJPlPnIls1FRnCmrz+yY0rmoOXYXVOwe+q5HsHBSFmsRokMxhXDO0W1n2f332BYbrbgQi8Gil0BbCMiDYAWA1gLjPPBzAdwAQi2glgvPrZFZbuVOYjPli11/SawF57KO+jcHl/1T5dmYLgPS2axZoPijfUGYSyt8P0a87Ak5OGhHXvmb3aR1R2uHjy5Jl5NzOfqf4NYeZn1fQjzDyOmQcw83hmPuqFfI1y+n+mgIlmq6YeO0okCUerQgySGoW4WOc47Nlz77gBjuYHAM3SmjaRPdq3tHz/5FG9fZ3JeEG6A0HQTwAD+olmVv8Hvz+c39V/1+fj5rdW2b9REBzEqDF0mtO7tXE0v0nDrS0269jKuvfOxDO6N0nrl9HK8v1u0KJZqqv5i1LQETrMhb/5yKpnQtbeY1i83dqc+V/mb2t0gRUEjzijRzuvRbCN1Tm89i2t+/inpsRWL/+7B8a4XkbSKoVIzDTavUdO1Fi+5853smyXU1BaafseQYiUISe39Wtgf35WT1fKia3m1j3sWAxaNTceBaSrI7d+nd0Lb6GRtEohHALNR24Taic4wVvSYqwX6RSBjdjJNmzoXhL4NEwnaqP82Jwo7vN7LsDDl52OlCi8c0mrFCKZR4vWXHCkG+kI7hKuV0kyEUxxur3J089H9nAlX7ty25nDMMt7YLe2uHvsqbbKDZekVQrhEOh95CZHT9Tg3ZXm7rJC9Pn1mH5eixB3/Oeu8zwr22zKz6xJt+pVZHeEaEeJvDR5uN/nByachtlTzrVVXqQkrVIIp11vnGh2Xyn85ettoS8SokvAb3vScHd6ol5DIMfe8SEnOzdh/e/bzg56PrDtNarDNSN64J83jjS8Pz0tBZ3UXv1bt2YaXgOYTz5fZcH7qWvbdNNzn/72fPTueJJf2u/HDcC5/TqFzNdJklYpaITjQ2z19xLJ4Dha8xaCdQLfldbp4YcOS4+Cy2cs4KiFyOZPwujyF64fjoHdjDfPaZ6W4pP3jJ7myiySuSTtZx34vYwe0Bkje3dwfbWyFZLjzQyCnVhDjeYjl4TRISohfP4wcZAr+Tr5gz2/f3R7f/HE+f07YcjJSsM90MZahsB+lJ3faadWzfHcNWf4Ppt1Fiee0d10pGDnNxuqz9ffw7UQSasUvt/eNOz2stzg6wO0OCgVNeHtj2yH/GPijhouk0f19lqEkLRKT8N5OrPA/eNPc7yMcH3siYBHLhto6577x5+GGwK+98xTOoQ1Wv7gznPRLFVpmv58zRm46LSMMHKxZ+b94M5zMaJ36I0e/3njSPzeZOW0E4P7Xh1PwqDubfH0VUMjzyxMklYpzN98yHes9QBr6prut6ynvEpRBi8u2GGpjEhGFCt2Hwn/5iTnJJdWfDo5sk8h8ht53Dve+RANkcwLnNGzHe6zIdO94wf49bQBoHWLtLA9jPSSh8oi85QOePqqoU166rbCywTcHcyCcFpXZ1di60lPS8XX947G+f07u1ZGKJJWKWiEM6egKYdQ6Fcx18VgXPZEJRJf7l4dzT1QwmnfLh9iHCHzjgv74s4Y9WZ6YILzoxYjrHyf+kvMGuo5vzkfN597ShMlGF4YevsPOdBjyAjNLdVslBFLJL1SsPLiBCoOq5PAr32/KyyZBO/46p7RpufC6UCYNXxn9mqPEeriqrYtvNzrqikXn96lSdrvLjH3kR/Ru71hOrP9JvbSwU02WzTM477xA3BKJ39PnUhGCr06aHmF33ljAE/8ZDA6GGyV2UZ9xvEwl5T0SgEI/fKs23cMu4qP+z5Lpz9xCbb3bTgjhZvPPcU8P5Mm86Mo+6VbYfygpo21htkiS4bxdzb1CmW+IpiS/c1FyiiqX4ZxWIf7xp+GHx4e619ewO/Sys909IDOyJs+Ea0CPMnCjWx6+4V9sf6JS8O6N1YQpWCBBVsKMe75H3yfSytrPZRGiCfOPzW0bditPsbDNieLA7l0cDf1v7lCAMwntM3mNII1t9odlw/tjrzpE9FOF7zO7hRJtN26g83hNLqixoDPaQiSXinEW6zzZOOyIcEbJDf489X+E6bahKvRm2JlrcLks3vZKt+Jpixv+kT85uL+pgHWrDD45LbImz4RQ3u0CzpKuslkNHTDqN6ONILW8wiYU4gk6KUur36dg7uH+ha16tLO798JJ7dr4ft85+i+AICMNuaL12KFqCsFIupFRIuJaAsRbSaie9X0aUSUT0TZ6t+V0ZKpoKwqWkUJIWidnoa86RN9nx+70p01B8Ew7WEaNE6bnrzM73OgQgH8t3TMaJOOwd2NF0/FAlZW5erp2jYdPzuz6T150yfiSoO9CPS43Wl2alX2oocuNj03zGSR2wd3novlj47zfb75vD7Imz4xogWP0cILCesAPMjM69R9mtcS0QL13IvM/LdoCsNg/PGzTdEsUghCEw8Shy0AedMn4sK/LMKBIOtA9EWOG9jFUIaz+4T2adfQz0GteXx84weXGsX2unkROz11vTK2SnjhYsLLL1RZTeYUInh3rFoQvrjnQnyx4WD4BanEUsiUqI8UmLmAmdepx+UAtgKInW9EiCnSvdgrWG1NbjynN/51a6ZPSeibCX1j2z7I5LSSXex6JlgNtmbWSFqpWfYTE2xI1DRHbeP7AV2D7yUQ6AByjoMxg1o0SzE1/YSj2288p3Gh357nrnRtz4pw8HROgYj6ABgBQNt/8h4iyiGimURk2BUjoilElEVEWcXFTVcl22Vn0fHQFwmuEehWGNiz1Va2RhOtcUlNIUUetVHXizZaN4Gcpe/968h99goA7k0kj7Kwx3GoxsZqg9a1Xfi28PYnNcf2Zy4P+/7/yeyF7c9cjlM62Qv9MKpvR7z/q3MMz2lmnFCT6Bqbpl2GFVMvCX6RxQe945kr8PSkoRg/SHH9jbXJZ8+UAhG1BvAJgPuYuQzAawD6AxgOoADA80b3MfMMZs5k5syMjPCWv+tZv68k4jyE8Hn/V+f4+XXHQq96aA/F5q9Fp2wcKRAGdFF6q5fqFqWl6RSX/vetpV8QygMpzCp/8KtzsO3p4I3tEz8ZjC1PXdYkfaS6tsBqg9SlTQtsfvIybHryMowe0Fgfq48rPa1xwjsc5w79/WYYrTky61Ssf2IC1v5hvOkkeSBpqSl+z1mP9hVaXSzXPC0FKSmE1286C1ufCl9ZuoUnSoGImkFRCO8z86cAwMyFzFzPzA0A3gQwygvZhOgxcVh39OxwkqUffDQ565SOWPfHCb6J0uHqIrNhvRonFc3aUqPN6M1663Y7iGsCRiRpqSkhN3FPSSGc1FzpFV9waqM55ZKBSi+1u85DJhSt0tPQOj0wdIV1jdatrX9ZRtV3qk+guclqnYxhPdv5KcdmqSno1Dq9iVK8TFX2J0XgtWWVtNQUtIxCOXbxwvuIALwFYCszv6BL17sqXA1AZn9dYkOMLK7RNlAfO7DpCto2LnppjDVYsRuIfrescYO6YtVj4yzdN8JsC0gDmqs9z/EBJoxubVv4evJ6Qrkz/vDwxabn1v9xAmb+UtmPYMjJbfHbi0/F6sfGoVdA/H4r6EdzgY34xmmXNvHI0lj80MXY8KdLMbC7ojiHGXxXVkxi5nIp/3u0b4kNf1Lecc0U2KJZqk85BuPJnw1B1h/GN1nMlkx4MVK4AMDNAC4JcD/9KxFtJKIcAGMB3O+BbElBsFW70URrT56aNMTXoF10umISXP7oJVj3xwmWDA3XjrQ2SaeZPZ746WA/3/OHLzs95L1d21rrUQczxwSu8WrRLBUrHr0Ef7l2mF96n86t8MGd9lc1t21h/lw7tGqO9LRUrHl8PObcdT5SUghdLNYpkMxTGhvuwI59mxbNTN0uWzZPRbuWzTB6QAaWPjLW0JV1SgTxoDRX4jYt0nwyaCYdq4OytNQUdG5tbf7kVNWUqJnDYsDy6QhRV4fMvAzGz2hetGURvEX7ETVLTcEpnVphycNj0UXdmaqN2sAdPVFjeG9aCvmCDLZKtzYEn3Fzpq+8ub8fjdvfXoMVu48gw2IjEAlrHh/vGxno6d7OOABfoFnoQgsro61gNtpolmrdlvW7S07Fi98pkYLPC9PDJ3CEsuThsWjfqpnji926tFEUn5Ww2HZY8eglvnc0xuaJIyZ5x0hCUFo2S0Vlbb2rZZwW4GLYu5N1U0ZaaqNS0P8mZ93uPxX1zX1jcNnflwCAn/22ZfNU9OrYEit2hxtN0x5WVrKuemycaRyhGbec5bRIPpZPvQQtbYQb10ehff66Mx2RoUvb9JDzI6Ew6qmf2qU1vrlvjOOb1hgp80QZKSR9mAvBGLd7P4O6t8Uvz+8TWg6TdP3ev/qeYeAet0YTvxra8L+bSW89Es60Mbeg0bVtC1PlYcUe3lzd4nO4zbJPbt8SHXRzKHaItCF3g8B35vRubUw9h5xAMy0GdnLiFVEKCU6gvbyNxTDNbo+IR/Zub8lU0CxgL+Mz1bACz1w1FH3VeQF977pPiDg1en51YT98NOVcXHRaRtAN1fUMVENUBJuInPv7C/HuHdF3nmuVnoYv77kQ//yF8cb0sYyTnZBom3POOqUD/nPXeXGxV4IVxHyU4Jx1SqMtddbto3yeH8FYcP8YXPHSUkfKf2rSELRr2Qz3zs72SzczkwTSOj0Nn/zmPKQQIaNNOjq3Tsf6fSUY1L0tbhjVC3+et803gWtksw9GSgr5Vr3O/f1oHCoNHQPrr9cOw03n9EaP9v6jiwX3j8FJqqLQj2IiYcH9YzDhxSV+ad/cNwYVNXV+prBFD16EtBSl7sE2nI9lnHBL1hR1f5Nw225ydp/wvaZiDVEKCcj0a87A2r3H8J+1B9A6PQ3L/ncsSipqMbRHY4Px+k0jkZqSgvqGBtz13joAwKs3jsSI3u3RvV3LsHaK+9ctmfjVrCy/tG5tW2BYz/ZNrrXTmzvrFP8f3HnqRiWaDTclhfD53ReYegh9c98YpKcFVxidW6db8jpp2TzVMHzCABe2aNTy1IemNjKHme05EA98e/8Yx1at9+3cCu/eMcqvIyTYJymVQkmFsUdLojB5VG9MGt4DEwZ39SmCngG/k8uHKstCVun2gg4V1TIUgf72r944EhMGd/XzIOrcOh2Hj1ejiwMhhDW9RRTchh9sXiHWmXPXeeje3vk5j1jB6f2ORw+IPMpBspOUSkG/i1q80iyVUFtv3ptv2TzVLxSDGdoE5k+GWVMIix+6GGP/9r2lazUl00nXA3/26qGoqKnDT4bZC9FshLYBS8eTmk6SLnrwIhwxcWeNJzJj1Cyx8MGLcCwBvl+hKUmpFOobvC3/DxMH4Zm5W/3SrhnRA5+uz7ech7JgRlEK944bgJcW7gxLljN6tsML153ZRIG0ap6KEzX1+GjKuSgqr8bvPlyP0QM6o2/nVvj87gvQwIw9h0/ggY83GOb75i2Zfp/HD+qC77YWgQBcPcKZiJDXn90LRMD/GISR6JfRGv2k0+ga/TNaAx58vwsfvAgFJbL/iZskpVKoa/BWK4wf1BUDu7XFTW8pwWEfmHAa7h57Kj5dn+83AvjugTHYe6QCd7yj2Ombp6Wgpq6p7PdPOC1spQAA1xisCJ5372hsLSjz2c8ra+oxQTUPaaaaEb07oFlqCjqc1BxVAWsaJliMPhkJqSmEG0b1Dn2hkDD0z2jtyURyMpGUSiGYPfuMHu2wMb/Ucl5/ufYMtE5vhtO6tsb0r7dh4bYinN+/E35xTm/cOzvbb9ctPRfqIk1qrmwvTR6OYT3b+8wzp3Zpg1O7tMGbt2Ti5PYtcFLzNGTlHcXDc3LQoVUzvH7TWb79ov/9y7Ox+/AJDHLIfn5Kp1Z+oYqvM9lS8qcGoQqM0FZ/BrqYCoIQWySlUuhjEJf9usye+HrTIfznrvNQWFaFj9bsx6vf7wKgLGmfv+kQBnZri3X7jqFnh5Z4+qutOHy8Gj8982SfXX7GLZl4/Ydd+OX5fdAqPQ0Du7XFrBV5OFRahV9f1B/XvrYcQOMio3/dkum3QYvZ7kv6Xnffzq1QU9+A0adm+K0AHjuwC8ZG+L04wWd3X4CCkqa7mk372RAM6NoaF8lEoCDENBQL8evDJTMzk7OyskJfaECfqXMBAF/97kJsPliK689uaobQrjHapnDvkRNYsKUQvxptPYDX6L8uwv6jlch+YgLaG0yOany8Zj+G9GjrmL+7IAiCHiJay8yZhueSVSnsLCzH99uLcWeQqIzzNx1Cs1TCuEHO2Md3Fx/H/M2H8NuLT3UkP0EQhHAQpSAIgiD4CKYUZNZPEARB8CFKQRAEQfAhSkEQBEHwEXNKgYguJ6LtRJRLRFO9lkcQBCGZiCmlQESpAP4J4AoAgwHcQESDvZVKEAQheYgppQBgFIBcZt7NzDUAZgOY5LFMgiAISUOsKYUeAPbrPh9Q03wQ0RQiyiKirOLi4qgKJwiCkOjEmlIICTPPYOZMZs7MyJCQCYIgCE4Sa7GP8gHoI6/1VNMMWbt27WEi2htBeZ0BHI7g/lghUeoBSF1ilUSpS6LUA4isLqeYnYipFc1ElAZgB4BxUJTBGgC/YObNLpWXZbaqL55IlHoAUpdYJVHqkij1ANyrS0yNFJi5jojuAfANgFQAM91SCIIgCEJTYkopAAAzzwMwz2s5BEEQkpG4m2h2mBleC+AQiVIPQOoSqyRKXRKlHoBLdYmpOQVBEATBW5J9pCAIgiDoEKUgCIIg+EhKpRCPQfeIKI+INhJRNhFlqWkdiWgBEe1U/3dQ04mIXlbrl0NEIz2WfSYRFRHRJl2abdmJ6Fb1+p1EdGuM1GMaEeWrzyWbiK7UnXtUrcd2IrpMl+75+0dEvYhoMRFtIaLNRHSvmh5XzyVIPeLuuRBRCyJaTUQb1Lo8qab3JaJVqlwfEVFzNT1d/Zyrnu8Tqo6WYOak+oPi6roLQD8AzQFsADDYa7ksyJ0HoHNA2l8BTFWPpwL4i3p8JYCvARCAcwGs8lj2MQBGAtgUruwAOgLYrf7voB53iIF6TAPwkMG1g9V3Kx1AX/WdS42V9w9AdwAj1eM2UNYHDY635xKkHnH3XNTvtrV63AzAKvW7/hjAZDX9dQC/UY9/C+B19XgygI+C1dGqHMk4UkikoHuTALyjHr8D4Cpd+ixWWAmgPRF190A+AAAzLwFwNCDZruyXAVjAzEeZ+RiABQAud114HSb1MGMSgNnMXM3MewDkQnn3YuL9Y+YCZl6nHpcD2AolzlhcPZcg9TAjZp+L+t0eVz82U/8YwCUA5qjpgc9Ee1ZzAIwjIoJ5HS2RjEohZNC9GIUBfEtEa4loiprWlZkL1ONDALqqx/FQR7uyx3Kd7lFNKjM1cwviqB6q2WEElJ5p3D6XgHoAcfhciCiViLIBFEFRsLsAlDBznYFcPpnV86UAOiHCuiSjUohXLmTmkVD2mribiMboT7IyboxL/+J4lh3AawD6AxgOoADA855KYxMiag3gEwD3MXOZ/lw8PReDesTlc2HmemYeDiXu2ygAA6MtQzIqBVtB92IFZs5X/xcB+C+UF6ZQMwup/4vUy+OhjnZlj8k6MXOh+kNuAPAmGofpMV8PImoGpSF9n5k/VZPj7rkY1SOenwsAMHMJgMUAzoNiqtOiT+jl8smsnm8H4AgirEsyKoU1AAaoM/rNoUzQfOGxTEEholZE1EY7BnApgE1Q5Na8PW4F8Ll6/AWAW1SPkXMBlOpMArGCXdm/AXApEXVQTQGXqmmeEjBXczWU5wIo9Ziseoj0BTAAwGrEyPun2p7fArCVmV/QnYqr52JWj3h8LkSUQUTt1eOWACZAmSNZDODn6mWBz0R7Vj8HsEgd3ZnV0RrRnF2PlT8onhQ7oNjrHvdaHgvy9oPiTbABwGZNZij2w4UAdgL4DkBHbvRi+Kdav40AMj2W/0MoQ/haKPbNO8KRHcDtUCbNcgHcFiP1eFeVM0f9MXbXXf+4Wo/tAK6IpfcPwIVQTEM5ALLVvyvj7bkEqUfcPRcAwwCsV2XeBOAJNb0flEY9F8B/AKSr6S3Uz7nq+X6h6mjlT8JcCIIgCD6S0XwkCIIgmCBKQRAEQfAhSkEQBEHwIUpBEARB8CFKQRAEQfAhSkEQdBBRvS6yZnaoaJlEdBcR3eJAuXlE1DnSfAQhUsQlVRB0ENFxZm7tQbl5UHz/D0e7bEHQIyMFQbCA2pP/Kyl7WqwmolPV9GlE9JB6/HtS4vrnENFsNa0jEX2mpq0komFqeici+laNm/8vKIvDtLJuUsvIJqI3iCjVgyoLSYooBUHwp2WA+eh63blSZj4DwCsA/m5w71QAI5h5GIC71LQnAaxX0x4DMEtN/xOAZcw8BEosq94AQESDAFwP4AJWAqPVA7jRyQoKQjDSQl8iCElFpdoYG/Gh7v+LBudzALxPRJ8B+ExNuxDAtQDAzIvUEUJbKBv2XKOmzyWiY+r14wCcBWCNEtYHLdEYlE4QXEeUgiBYh02ONSZCaex/CuBxIjojjDIIwDvM/GgY9wpCxIj5SBCsc73u/wr9CSJKAdCLmRcD+F8oYYxbA1gK1fxDRBcDOMxKvP8lAH6hpl8BZStLQAlG93Mi6qKe60hEp7hXJUHwR0YKguBPS3XnK435zKy5pXYgohwA1QBuCLgvFcB7RNQOSm//ZWYuIaJpAGaq91WgMdTxkwA+JKLNAJYD2AcAzLyFiP4AZZe9FCgRWe8GsNfhegqCIeKSKggWEJdRIVkQ85EgCILgQ0YKgiAIgg8ZKQiCIAg+RCkIgiAIPkQpCIIgCD5EKQiCIAg+RCkIgiAIPv4f8FOBVzpIQfkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import gym\n",
    "from dezero import Model\n",
    "from dezero import optimizers\n",
    "import dezero.functions as F\n",
    "import dezero.layers as L\n",
    "\n",
    "\n",
    "class Policy(Model):\n",
    "    def __init__(self, action_size):\n",
    "        super().__init__()\n",
    "        self.l1 = L.Linear(128)\n",
    "        self.l2 = L.Linear(action_size)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.l1(x))\n",
    "        x = F.softmax(self.l2(x))\n",
    "        return x\n",
    "\n",
    "\n",
    "class Agent:\n",
    "    def __init__(self):\n",
    "        self.gamma = 0.98\n",
    "        self.lr = 0.0002\n",
    "        self.action_size = 2\n",
    "\n",
    "        self.memory = []\n",
    "        self.pi = Policy(self.action_size)\n",
    "        self.optimizer = optimizers.Adam(self.lr)\n",
    "        self.optimizer.setup(self.pi)\n",
    "\n",
    "    def get_action(self, state):\n",
    "        state = state[np.newaxis, :]\n",
    "        probs = self.pi(state)\n",
    "        probs = probs[0]\n",
    "        action = np.random.choice(len(probs), p=probs.data)\n",
    "        return action, probs[action]\n",
    "\n",
    "    def add(self, reward, prob):\n",
    "        data = (reward, prob)\n",
    "        self.memory.append(data)\n",
    "\n",
    "    def update(self):\n",
    "        self.pi.cleargrads()\n",
    "\n",
    "        G, loss = 0, 0\n",
    "        for reward, prob in reversed(self.memory):\n",
    "            G = reward + self.gamma * G\n",
    "\n",
    "        for reward, prob in self.memory:\n",
    "            loss += -F.log(prob) * G\n",
    "\n",
    "        loss.backward()\n",
    "        self.optimizer.update()\n",
    "        self.memory = []\n",
    "\n",
    "\n",
    "episodes = 3000\n",
    "env = gym.make('CartPole-v0')\n",
    "agent = Agent()\n",
    "reward_history = []\n",
    "\n",
    "for episode in range(episodes):\n",
    "    state = env.reset()\n",
    "    done = False\n",
    "    total_reward = 0\n",
    "\n",
    "    while not done:\n",
    "        action, prob = agent.get_action(state)\n",
    "        next_state, reward, done, info = env.step(action)\n",
    "\n",
    "        agent.add(reward, prob)\n",
    "        state = next_state\n",
    "        total_reward += reward\n",
    "\n",
    "    agent.update()\n",
    "\n",
    "    reward_history.append(total_reward)\n",
    "    if episode % 100 == 0:\n",
    "        print(\"episode :{}, total reward : {:.1f}\".format(episode, total_reward))\n",
    "\n",
    "\n",
    "# plot\n",
    "plot_total_reward(reward_history)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2acaed03",
   "metadata": {},
   "source": [
    "## ch09/reinforce.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7d956534",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode :0, total reward : 13.0\n",
      "episode :100, total reward : 48.0\n",
      "episode :200, total reward : 36.0\n",
      "episode :300, total reward : 26.0\n",
      "episode :400, total reward : 48.0\n",
      "episode :500, total reward : 33.0\n",
      "episode :600, total reward : 193.0\n",
      "episode :700, total reward : 58.0\n",
      "episode :800, total reward : 124.0\n",
      "episode :900, total reward : 200.0\n",
      "episode :1000, total reward : 108.0\n",
      "episode :1100, total reward : 200.0\n",
      "episode :1200, total reward : 200.0\n",
      "episode :1300, total reward : 176.0\n",
      "episode :1400, total reward : 122.0\n",
      "episode :1500, total reward : 200.0\n",
      "episode :1600, total reward : 196.0\n",
      "episode :1700, total reward : 200.0\n",
      "episode :1800, total reward : 200.0\n",
      "episode :1900, total reward : 170.0\n",
      "episode :2000, total reward : 200.0\n",
      "episode :2100, total reward : 200.0\n",
      "episode :2200, total reward : 200.0\n",
      "episode :2300, total reward : 200.0\n",
      "episode :2400, total reward : 200.0\n",
      "episode :2500, total reward : 200.0\n",
      "episode :2600, total reward : 200.0\n",
      "episode :2700, total reward : 200.0\n",
      "episode :2800, total reward : 133.0\n",
      "episode :2900, total reward : 200.0\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA5TUlEQVR4nO2deZwU1bWAv9OzMTAzDDADDJvsAsqmI6vggiLgnhiV5EXjRjS4JMYkbknM85mY/WleXgw+fWqiJsb9GbMYNUETN1BEVBBQEJBV2WSHOe+Prh56ZnqpXqqruvt88+vfdN+qunVuLefce+6954qqYhiGYRgAIb8FMAzDMIKDGQXDMAyjGTMKhmEYRjNmFAzDMIxmzCgYhmEYzZT6LUAm1NXVad++ff0WwzAMI6+YP3/+JlWtj7Utr41C3759mTdvnt9iGIZh5BUisjLeNnMfGYZhGM2YUTAMwzCaMaNgGIZhNGNGwTAMw2jGjIJhGIbRjGdGQUR6i8jzIvKOiLwtIlc56Z1F5BkRWer87+Ski4jcLiLLRGShiBzhlWyGYRhGbLxsKewHvq6qw4BxwGwRGQZcCzyrqoOAZ53fANOBQc5nFvArD2UzDMMwYuDZPAVVXQusdb5vF5F3gZ7A6cCxzm73An8HvuWk36fhWN4vi0itiDQ4+Rhpoqo8vmANJx3Wnfblpbz+4WaWrNvOoK5VPPL6ai6Y2I8Fq7awcPUWTh/Vk7KSEKUh4fCeHVm+8VMemb+aDzbt4LUVm2lS5Q+Xjuf+lz+kS1U5Y/t15id/XUKPjpX87d31fLpnP6P7dOKovp3ZtXc/T775EZt37muWZeaY3jz46qrm3x3KS6htX87arbuoLCuhf30VQxuqeWje6uZ9yktCVJSG2L5nf4tyjezVkb0HlKYmZcn67dRVVbBt9z727m9qcw26dCjn4x17OfKQTsxfuTnutappV8q23fspKxH2HdDm/8koCQnty0vYvjssY8/aStZs2dVin4rSEHv2N3H8kK68uGwT9VUVbfaJvk6rN+/ihaWb6NWpktWbD+43pHs1i9dtB+CwHjU0dKxk5cc7WLrhU7pWV7Bh+x4AqitK2bF3P5dM7s+v//F+i/wnDOjCv5Z/DMCgrlWs27qb7Xv209CxHScO68ZD81ZRFgpxeM+OvLbiE8YP6MILSzcB0NCxHRu372F/U8vrUllWgqJMHdadNVt28caHm2lSGN+/Cx0qSnlu8XoGdq3ipMO6s2jNVp5fshEI38djBtfz6BtrqKuqYMGqLW3KXNOulHZlJWzYvofSkDDt8O70r+vA39/byMbte1i7dTd9u7Tn2EO78od5q9ix9wAA5aWh5uehrESYMKCO5Rs/ZfXmXYzv34WX3g9fg+417ehf34E9+5tYu2UXDbWVzF+5mXZlIXbvO/g8RZ4jgP71HVi7ZTczhjfwyOur6d25klWf7GJEr45MGFDHHf9Y3uL6DKjvwPKNOwA4qm8nXluxmZNHNPDHhWupblfa/OxEPy9nN/Zm06d7mp/vVz/4pHn7aSN7cPWJg+lb1yHmM5QJkov1FESkLzAXOBz4UFVrnXQBNqtqrYg8Bdyqqi86254FvqWq81rlNYtwS4I+ffocuXJl3DkYBjBvxSecdcdLfO7IXvz4cyPpe+0fXR234taTXe9rGEbuuWRSP244eVhax4rIfFVtjLXN845mEakCHgG+qqrborc5rYKUrJKqzlHVRlVtrK+POUvbiOJTp4a9bttunyUxDLhgYt82aQtvmppyPvddOCYL0uQ3rVtr2cJToyAiZYQNwv2q+qiTvF5EGpztDcAGJ30N0Dvq8F5OmpEBIRG/RTAMwwO8cvJ4OfpIgLuAd1X1Z1GbngTOd76fDzwRlX6eMwppHLDV+hMyJ2IUmmzZVSOgWLUlPbx6p70MiDcR+CLwlogscNKuB24FHhKRi4CVwNnOtqeBGcAyYCdwgYeyFQ2RhkJT2/5Xw8hb0mkAl4SEAx65XPzAq7J4OfroReJXAqbE2F+B2V7JU6xEXh5NrevGMDxBYqgEyZGLs9BaJF7ZN5vRXOA0u4+aYOXHO3yWxjDaUmjK2gvOG39ImzSvRo6aUShwIi/cqys+4Zgf/91PUQwja8RqcRQyleUlbdK86lMwo1Dg5KppbhhuiPU42iOanFhG0NxHRlqE7IUzjLwnluG0loKRFtZSMIJOOq4ge6zzcJ6CEQzs5TGCTjrPqBcKsZ8HcYSyRaxL5NWQVDMKBY7NaDZygdvHLMhP47CGGr9FiIu5j4ysYX0KRi4oscqHp8RysZ3d2DvGnpljRqHAKbahe4Y/hAqg9hHkCZ6xbO7kwd4EBDWjUOBYBc7IBW5tQraGpAZZgXtBLl9jMwpGTGwtBSMVMum7CkprNtAxI3NYuzOjYBhGxnStrvBbhILGWgpG1jD3kZELnr5qEtdOH5J0v1jzZoIyJDXILYVcvsdmFAzDyJj25aVcesyAtI61ektyculi83I9BcMw8oyQeBdTJ6icPqoHEwZ04bnFG5LvXARYS8EwjGb8CIuSzjmzabdG967lnKP6ZDHH7GPuI8MwXPHgJeOymp/Xuidb+XuxlkCg+xRyeC4v12i+W0Q2iMiiqLTfi8gC57MiskyniPQVkV1R2+7wSq5iIyjD/QxvGD+gS1bz82Nggj2hycnlffGyT+Ee4L+A+yIJqnpO5LuI/BTYGrX/clUd5aE8hlH0nDyigT8uXBt3e7gSkdsqs98j5CLuqwA3FHLq1vOspaCqc4FPYm2TcAnPBh706vxGGL9fOCNYTBjQhaGJAr/lwH9053mNGWcTZAWeKT1rK309v199CpOA9aq6NCqtn4i8ISL/EJFJ8Q4UkVkiMk9E5m3cuNF7SQ2jwIin9xsP6cQvP3+E5+c/cVi3lvIEpOaSTp/CF8bmpoO6GDqaZ9KylbAW6KOqo4GrgQdEJGZ1RlXnqGqjqjbW13sTEKqQCMj7ZgSIeM/Ew5dNaKOwjcScN75vTs6Ty77BnBsFESkFPgP8PpKmqntU9WPn+3xgOTA417IVIkEeUWH4g58VhSAOfDh4PVJ/WXIVHLbQWwonAItVdXUkQUTqRaTE+d4fGAS874NshlHQiPOXLl06lGdRmgwISGUnV8q6UIakPgi8BBwqIqtF5CJn07m07WCeDCx0hqg+DFyqqjE7qY3UMPeR0ZovH9PfbxECSTqtai/6Q/x+Zz0bkqqqM+OkfylG2iPAI17JYhjGQU4Z0YPLH3gjrWMDUkHPKol0cE27Urbt3h93e66Wuy1095GRQ4LowzXyAy9W9sqWcvNikZ10csxZn0IhdzQbhuEfmSjlqcO6eRJeIp/xoqWQrdXp0sWMQoHjt3/SyF+iH533vz+DX3/xSN9kyQWxDF4yE+jl+3XayB7eZZ4AC51tGEZSQqHshILIXkC8LGUEGWl2rybeLb55GmUlIZ5a+BFNWiBhLgzDMLzCk5XX0jjGC1UtCO3KSigJCaUlIc/OEw8zCoaR5/zgM8Nd75upcim6LoUclvd/YsSEiri0rE/BMAqc2celt3RlLKoqvPECe6GIgtLHdeyhB0dWRURK1eC9cv2UrJanprKsTdoBZxm80lwNc8KMQsGye98BwGLVB5XykhK/RQCgvrqiTVqilkdQRh9lKsWcLzYycWBma1F0q2mXoRQtiWVgIkujhswoGJnwt3fWM+Tbf2bBqi1+i2LkId2zpOyC0iqIRXlpiEO6dGiRFsvQJDM+2bSREb0f67q1bimUeGgkbPRRAfLC0nBI8QUfbmbiwDqfpTHyjcjEsFhqJxUdmPvlelIjII2eKOIr+uj5EM9fc6xnLkOwlkJB8dzi9fS99o+8v2lHc1rgnnsD8GZGbizKS1q+4l7X3v/xjWOjztX2ZPFm5j51xdFeiZSUiJjpuMZiHXHZsen1FyWq/EcbhX51HWK6/bKFGYUC4v/eDC+zuODDLUBwFi8x/KOuKstRTZPozWiXTCpP3+E9O6Ynj8ekYygqStNTq5H3Nfq6RVZhy2XlzoxCAWKtg+ATPNdF9vGyThJLWd/xb8GYcZ1unKJYRz02ewIPXDI2M4FSxPoUChhrKASXspLg3pyIvo3V0kytT6Ftr4KXz2SvTsnXNn7ma5Ojop5mbpljGaey0jSNQozDula3o2t1O/4wbxUAwxKtr50lrKWQ59z81Dv82/+80iLt0z3hh74YaqP5Ssf25dw+c7Tny1+2fgRyGjU3y6eKdoWl+2gP6lbNkYd0apEWuSax3pdUz3PbuaPSHr0VkSOR23eoGQUjGXe9+AEvLtvktxhGqqhy2sge1FV512HoBan42LNtfnp3bp9wu9eVoG41be9V63OePqpn2vknakVZn4KRFcx9lA8ErzmXLeUaMwR0drKOSapj91uXM9mIMLetLC+NUy7eaS+X47xbRDaIyKKotJtEZI2ILHA+M6K2XSciy0RkiYic5JVcxYa5kLwjln/X7VDBfLgtmc9TyK4Gi36WYz3XQxuq08r34JDUtA7POjGvWg5l87KlcA8wLUb6z1V1lPN5GkBEhhFeu/kw55j/FpFgxAHII1o/TNZQ8JYvjOvTJi3w1zxDAVNRnLluqYoIl0zql/bxpSVt1WEuDYWb65WLS+qZUVDVucAnLnc/Hfidqu5R1Q+AZcAYr2QzDK9IVRGmo3SSRUXt3fngKJxsK7VUyhd4A9mKH581IuF2t2VP95IHZelcP/oULheRhY57KTIMoCewKmqf1U5aG0RklojME5F5Gzdu9FpWw0iJXPidZ45p20KJ5oYZw9LPnINKLZYS/MOl4zPKO3vNh2wMJ235u1tNu5RnCmfT6DZfmhiXKFcz4CH3RuFXwABgFLAW+GmqGajqHFVtVNXG+vrsLyxuGJkQtM79bMtzWA/3M4+9nFEfTxmno6QTSZmOMk43kmyiy3XqyB7MGN6db5x0aFp5p0JOJ6+p6vrIdxG5E3jK+bkG6B21ay8nzciEoGmoIiDVK+61z7p1/l48EbeceXicc2e3cAHpB25BNmvwiVqZ7ctL+e8v5GbGdk5bCiLSEPXzTCAyMulJ4FwRqRCRfsAg4NVcymbkP9ELp/iF29pxUNYliEWqsn1h7CEx00tLQkwa1DJKr5urU14SYkzfzinJECEVyWMp9ERF97qKFXl0/K7KeTkk9UHgJeBQEVktIhcBPxKRt0RkIXAc8DUAVX0beAh4B/gzMFtVD3glW7EQDjIQXOWTba6dPsRvEQqscZZZYX76uZH85qI04vbkfNRS/G3J7GM2ZkE3y5HmcdnGM/eRqs6MkXxXgv1vAW7xSp5ipHjMQXBIdYRKPhrtwd2qeG/9p0n3OyHNEB5BUY7Fis1oNowsko1hhRdM7Ju5IB6SVu3fwdVY/Fb7jO5Ty4UTW84/iGdKM/fKxc8gnY7zId1Tn1Dnd8h7MwoFjNW4ck823ufvnnpY5pnEwY3CSTQkNRcI0qIF9dhXJvKdU4cl1PjHDE69P8mNAUm6HGfU96tPHNwmcWDXKt78ztRURfMVC52dp7y0/GMObV0LMSvgOyHXHc0t/7sl1Q7YfHRPpcPMMb2T7xSHdFp3nTuEI7ZGguTd+pnhnBtj/oiIuL4HQblT1lLIQ1SVmXe+zOfvfLnVhpY/C6vTMz/4fAzF0K4se6/ZQ5lOHssCXg+ccvPcxp2nkKFq/fJkd0tpvv7tE4HwUNEVt54c0yCki9+vrRmFPGbxuu1+i1DwnDc+9nDLeJwxuu1E/MU3T2+TFpRaYSw8jfLp2Yzv1FVprFNcMrl/yvm0zbdlzgEefRwTMwoFTr49kEHjhpOHpn3sqSN7JN0nG7dnWA/3C6+kojrj7RsEl1Q8GdJ63hNPac6IVK53UN5VMwqFhN/tzjwinY5JN0S7Pm47ZxTLbmnbSsg2A+qrWHrLdKZ6vIpbrohvjBIck4Nn38tzRI8489vta0YhD3FbowhK1MUgkuJ6LHG54viBLX5HZxsKScxwzJD9Gc1lcc4TNLyONBoE2i7ek5zSbD2QWSA/niTDyDLZGgte3koZu803skBPrmMfuTzKgzyzSzoyzBje0OK3uyGpmRU2lccsCG45sCGpBY3fzdAgk62aejq5vHTd8TR0rEy+owektB5CnH07VHirNpIZ1gcuGcvHn+5tkda7U+L1m1+74QQ6VpbFPl9q4nlCdJn9buFbS8Ew0qRflw5t0ty8zrk0CF5UDDpWlvH5sekNwXQrTiyjHUnqUN7WKCXrbK+vrqC8NDN1F7mWyYYYt5Y8XgXk+hlDUh7dlguspZCHBKORGTxq2sWuCWZCvFrbuP6dmTCwjnkrN7dILylJcfF4j++mV66enrVhw+aHUhNJFOYiy+G6W2X38nVTUpp3IkBFWeyVhctKQvSo9afFmIi4RkFErk50oKr+LPviGNkmCP7fXJHLF+zQbrFj2pSFQvzr2uMD1XGYKpFnxo0bwwtXUiZXrn99let90zHI3Tu2S/mYqopS/vLVyZz0n3Pj7iME511NZPKqnU8jcBnh5TF7ApcCR3gvmmEEn9YvcklI6FFbSdea1JWHF3itZ4Jm+s4bfwi/nzWuRdrfrj4m4TFuBwdkMvGuTUgaoCGOgfG7LzCuUVDV76nq9wivgnaEqn5dVb8OHAlkb0634RlBe2G95MpWQ0P9IuUWQo5rh34onBe+eRwnHdYtpfMnDUQXp1otIozt36VF2sCu7lsPAB3KD7p7vLo9dVUVnHRY98C0DqJx0/brBkR39e910gyfcOs3vfbRt+K6OQqNVBdc94pQwNxG6Ujj5ulKxXffu3N7Bnat4i9vr0+4X3lJiL0HmuIKHe3uqW1f7vr8cYlThGyGro6X1+TBdS23SX64jyLcB7wqIjeJyE3AK8A9XgplpEes5u2S9RYfKZ/o7ridOrVv2Wn+8KXjmfuN41LOLxM9k87aB3HlcCHIU1ce7SovQZg8qI5fzBzt7uRJ83O5n+tWjvurHpS5CdEkNAoSNmX3ARcAm53PBar6g2QZi8jdIrJBRBZFpf1YRBaLyEIReUxEap30viKyS0QWOJ87MilUsRLEByzfcasIzjqyF3ed35hy/q3v2GOzJwDwxysncfeXDubX2LczfbokHoufDySqhXerPuhjT2ZERMRVbCk/qPJ4HofXJJReVVVEnlbV4cDrKeZ9D/BfhI1KhGeA61R1v4j8ELgO+JazbbmqjkrxHIbhKxFD3KNjO6YMTd2rWtMu/ApeM3UwQxtqmucw9Kit9GQ0VbYmRiVS2n+6ahIbt+9JOc+QU0WtrwqGKxDSG+J66oge/O8/V7Bg1Za4VzvWfQhKpc6N++h1ETkq1YxVdS7wSau0v6rqfufny4Q7sY0Uiffo+D0T0ldi1EBvzCDCKcBvLhqT0fFu+Nb0Idx48lC+cuzAtIxKtnjgkoNLbKaiB2M9c0MbapjcKuCgmyyr25Xxo7NG8NuL01/uMxVSVcFu365QSPjiuNTmb0RfR7+X43TTzhkLfEFEVgI7iAypVR2R4bkvBH4f9bufiLwBbANuVNUXYh0kIrOAWQB9+tggKCM+w3t2zOj4SYPq2XegKUvSxKZ9eSkXT8o8hn+mDIwxvl8EHp89kY+27MqZHGc3hldQi6WwveqI9UIHB6POnx5ujMJJ2T6piNwA7Afud5LWAn1U9WMRORJ4XEQOU9VtrY9V1TnAHIDGxsZ8vvaGx+SixtWsqPweXO4Ro3rXMqp3bZt0r1+8igRRX3N5qdMtZ0T8ZNFrow1d3ow+UtWVqroS2EX4GkU+aSEiXwJOAb6gjsNOVfeo6sfO9/nAcmBwuucoVFZv3sllv53P7n0H/BYlL0ikPNyupZzvtFY0rpa6dPF6R0beejUC97aZo7zJOIpsh8SI5pQRPbj46H5cPyO2C7P1fRA5mOb3bPikRkFEThORpcAHwD+AFcCf0jmZiEwDvgmcpqo7o9LrRaTE+d4fGAS8n845Cpn/eOpd/rRoHc8t3uC3KL6Qqh4/ok+nuNu+/5nh7s6Z2ikLikR9VBdM7McXxvZh1jHu1jROVf8GMWhgKi3PspIQN54yjI7t3cfjGtq9houP7scvP+9vwAg3Hc03A+OA91S1HzCFcCdxQkTkQeAl4FARWS0iFxEejVQNPNNq6OlkYKGILAAeBi5V1U9i5WsYbhjXvzMlIeHsxthjGboFJAxFMu69cAzfnHZo2sd71SDqUFHKLWcOT3n4Za4baFdOGZTR8bl06YRCwo2nDPN96LGbO7rP8fWHRCSkqs+LyH8mO0hVZ8ZIvivOvo8Aj7iQxTBi0lrXXDixny9yZJtjBtdntHRoUPzU6fDjs0awIWpoazplufpE80KnipuWwhYRqQLmAveLyG2ERyEZAaNI3OQpccXxmdUUk9Hcz+zpWXKLF4YknTH4n2vszezjvItp5Uai+y4cw+Bu4ZFZ2bzHrfMK0vPjxiicDuwEvgb8mXAn8KleCmWkRz7XCt2QzovTu3P+zwL2DQ80VRDn0iSSafLgep66YhIje9dyy5nu+qHyHTdG4VxggKruV9V7VfX2yEghwygmenVKv/Nz1mT/5yL4ycyj+lBXVcFpo4IZmiIR5aUhnpg9kfEDuiTfOUW8HAGVLm6MQh/g1yLygYj8QUSuEJFRHstlGHlFMtddvKGJxULfug7Mu/GE5hXb0iFX6rNDRTh09jdOSr+DP1WC5Pp1M0/hu6p6PDAMeAH4BjDfa8GMtiTzywbpwfKbWMMHpx/ePSv5RHPqiAYATnH+J2LCgC58ZnTPlGXINUHvJ8nWcx6vkl7qBGE6PQetmunDw8/NaSOD81wkHX0kIjcCE4Eq4A3gGsLGwQgQW3ft4/kinb/gJ4O6VbPi1pNd7fvAJeOS75R1wprvyEM6MX/lZt/j6gQRPy/JgPoq189PrnDjPvoM0AX4G/Ao8ISqrvVUKiMmkQ6xWDWcyx94nY937G27oYBIpNBM1yXGLo//RK8OGOTn1Y376AjgBOBV4ETgLRF50WvBjNR4f6ONEjayQxA7P70g16W8euqh3OpyJr2fuHEfHQ5MAo4BGoFVmPvIF4ISbz1fybR2FuTanRcEzdVULMbKb9y4j24FaoDbgaGqepyqfsdbsQwjM7xQZ4Wgk1pfl0gYkKp2B+uHh/WoAeCEoV1zJVZKZGuuw4nDwutXDGuoyUp+hYIb99EphA3Cx6q6z3uRDID31m9vUzNq7lOwFkNWue3cUXG3BauunDrxDNnEgeEx99dMPZQl/zGN9uUHjcLArtUsvnkap4/yfkTMmH6dPT9HPE4b2YPFN09jULfqnJ87iJP4IriJknoqsIDwbGZEZJSIPOmxXEXN35dsYOrP5/Lw/NV+ixIovHqNSlyGKg6YNyUlWst+zwVjeOffT0JEqCgtabN/u7K2aV5wf45WWYtHrsoZIR+qc27cRzcBY4AtAKq6ACiMaGMZsn7bbk/yXbbhUwDeXbu9RXqiFsKaHK6OZeQfrVsMZSWhFq0Dv0i2CI0bxvfP/kzjYsbNHdmnqltbpeWDwfOUZ95Zz9jvP8vfl3g3N8DcRIk5qu/B9RJy0RwviD6FPG7txOK1G07gfy9IeQn5hBw9sA4g5bDghYIbo/C2iHweKBGRQSLyC+BfHssVeBas2gzAojWt7aV3JJqnUIzcd+HYjMJKRyiG61lIxiC6LPXVFVl3Ad18xuHM/cZx1LYvz2q+0QT5frgxClcAhwF7gAeBrcBVXgqVT6SiUFSV/SksBN+69msth5ZUlpfQozb2YjmxXrpMWxNBfpGN7FFeGvJ9oRs/cTP6aKeq3qCqR6lqI/AbwiuoGSnyu9dWMfCGP7F2qzv/fzwjUKzKyY9y5/u1tmqEkSpxjYKIjBCRv4rIIhH5DxFpEJFHgGeBd9xkLiJ3i8gGEVkUldZZRJ4RkaXO/05OuojI7SKyTEQWioi/C5V6wBML1gDwwaaWs49P/+U/Oew7f3adTzG4OwyjEMmHdzdRS+FO4AHgs8AmwsNSlwMDVfXnLvO/B5jWKu1a4FlVHUTYwFzrpE8HBjmfWcCvXJ7DFw7OGcicN1dtYcfeA+7PmQcPViYc0afW9b6Ffi0ype0KX/nb9Cmkex3kFmgio1Chqveo6hJV/U9gh6p+U1Vdj8NU1bnAJ62STwfudb7fC5wRlX6fhnkZqBWR5PGIC5B44QWKpU/hjDTCS2fykhXyVS3EsgVZoRYCicZctROR0RysbOyJ/q2qr6d5zm5RUVbXAd2c7z0Jx1WKsNpJaxGRVURmEW5J0KdPnzRFyA+ia0bvrd/OX95eD9hL4Qa7Ri3J5xaCkVsSGYW1wM+ifq+L+q3A8ZmeXFVVRFKqzKjqHGAOQGNjo28VoWwpnR179rva7+m3DtrGIDejH7xkHDPvfNmTvMOKzb/C57NiLZZWZtDJh/sQ1yio6nEenXO9iDSo6lrHPRSZ/bUG6B21Xy8nraBp3ekMcPNT4X78eIYnyI/V+AFd6F7TjnUZzPb2TPWmkXG0Ky8fXuhk5HMLqhCuf6UzpyLIE+P8kOxJ4HzC0VfPB56ISr9cRH4HjAW2FvtiPtEtgnyupaZKrl/9YgjJXEjPTz6X5fRRPdmwfQ/nj+/rtyhx8dQoiMiDwLFAnYisBr5L2Bg8JCIXASuBs53dnwZmAMuAncAFXsqWLVKbvNY2LVHNbe7SjTHTg/5K5FuNzm38nXxWRkYwKAkJlx4zwG8xEuKpUVDVmXE2TYmxrwKzvZQniCRSNO9v3MGBJuXK373Bnn0HZ0L/c9mmXIjmG3GvSIwNrQ1teWmIvfubUlLgU4d147JjB/Crvy93fUy+UAytICO7xDUKySaPZTD6qGhJx5/70ZZd/HFhSy/ao28Eu6vFDz0UubQTBnTh70tit7DiUVoS4lvThiQ1CvnWAoqFtXWMZCRqKfw0wbasjD7KZ9J5uVJ1H0E4vk/Rkc+9oQEjaEtqZoI1enKDH6OPjBTIx1c6Yz3k0dufqVj52KcQcR9959Rh3PnC+xw3JJhLbKZCAdm5QOKqh01EDheRs0XkvMjHa8HyhXguhd37DvCDp99lV1T4imJ5mAu1RveLmaP9FiFtetRWctu5o3O+0piRf7hZjvO7wC+cz3HAj4DTPJYr+CTR8P/7zxX8eu77zJn7fptt23bt4/Znl9LUpC2y2bJzL794dmmLfQtUvyYmzrV1Y1O9NEjDe3bkplOHeXcCDyjK58fICDejj84CRgJvqOoFItIN+K23YgWTnXv3c/2jb3H80G5J993nrJuwv6nt+gk3PfkO67bt5rAeNfTsVNmcfsPji9p0KucjuVREcTt/PWiVFUtLzyhu3BiFXaraJCL7RaSG8Azk3skOKkR+8pf3eHzBRzy+4COunDIo4b4rPm47UznCjr3h0Bb7Wi24szNGyIs1m23tZTeYwo5NIV2WVCobxx5az+jenZLvaLTBjVGYJyK1hENpzwc+BV7yUqigsn33vjZpqvDK+x9TU1nG0Iaa5vRHX48/bFRafI8Oo9CWs+4o+pVPjQwoRPeRG0N3zwVjPJejUHGz8tpXVHWLqt4BnAicr6p5Mds428R7wc6Z8zLTb3shrTyT1XD3HSjE1zqYdKws45qpg+Nuz+fhnfkruZFr3HQ0Pxv5rqorVHVhdJrRkg827WDRmq2u9i3UUTqZEk+BudHJiS5pMqX+5nencvnxid2ChlHoJJrR3A5oTzhuUScOvqs1hNc5KGriqZfjfvL35MdGKaemArQMQSiS1YwNIz0S9Sl8Gfgq0AOIDmmxDfgvD2UKLNlSdlt3HeybuHPuB9nJ1DAKHIvjlBsSzWi+DbhNRK5Q1V/kUKa8IpXHtPUz/e7abSxZvy2r8gSD3L28pieKjzzu2skL3Mxo/rWIXCkiDzufy0WkzHPJAk42Hszbn1vGojX5axSumz7EbxGayccQFLmgkIzmGaPCXusuHSp8lqSwcTMk9b+BMuc/wBeBXwEXeyVUUMk0Smah1XDilyezgsbL143iNxdDbArh2bv8+IFcPKl/cQaJzCGJOppLVXU/cJSqjoza9JyIvOm9aIVH8egr/wuaz8NHvaAQnj0RMYOQAxK5j151/h8QkealgkSkP3Ag9iHJEZFDRWRB1GebiHxVRG4SkTVR6TPSPUdOSeFt27677YzlNLMKBLl22ZwwLHl4kUQUmplobwrS8IBE7qPIO3QN8LyIRCK79SWDpTJVdQkwCkBESoA1wGNOnj9X1Z+km7fnZKi031mbuP+gUIanelWMn3xuBP/35kfeZJ5nvP7tEykrcW/mrOFkuCWRUagXkaud778GItWSA8Bo4PksnH8KsFxVV1pzH15YWtjLbGZKRanVjCN07lDutwhGgZLIfVQCVAHVhI2HOJ9SJy0bnAs8GPX7chFZKCJ3OxPmAksqrhNVLcgO0CDY8cK7qtmlEJ87w1sStRTWquq/e3ViESknvC7DdU7Sr4CbCb/nNxNeDvTCGMfNAmYB9OnTxyvxYpLO67Xp0z30u+5pbj7j8KzLY0QhCX8ahuGSRC0Fr9+r6cDrqroeQFXXq+oBVW0iHJE1ZphDVZ2jqo2q2lhfX++xiMlJZihWbNoJwKOvr/ZemALB5hxkD3PLGqmSyChM8fjcM4lyHYlIQ9S2M4FFHp8/I1J914rp1TSHRXAw95GRKnGNgqp+4tVJRaQD4TDcj0Yl/0hE3hKRhYSX/fyaV+dPxEX3vMbpv/xnzG3pvGCFMqIolyQyuL+9aKwn+RY61voy3OJmRnPWUdUdQJdWaV/0Q5bWPLt4Q1bzi9iEYmrGZ1o7TXT40YPqMsrbMIzEuIl9ZDjE0lXJ9F8kNEYhmoQgGLrW198aZoaRGWYUPKYYlVSmxiKdw1sfEgB7ZRh5iRkFj4n0KRSTkvKzczMSG6ckVEQXPAFFWCcxMsSMQgpE6zq3Kqcp0qdQgA6kbJbooqP7UV+deUjkH352BF87YTDj+nVJvnMxUXiPn+ERvnQ0FxPFWFNLpcwXTuxHRVmIa6Yeyo49+/nda6syOnfnDuVcdULsdZaLWi8W44NopIUZhQxJtsZCsyulqDVSfOqqy/nKsQOB/Op/ySNRDSMlzCikQLQicNtHEOlTWLJue/YF8pl41yBT5R5k+3nOUX14+6NtXH3iYL9FSY0gX1QjUJhR8JiIgty6a5+/ggSU6L6WdFa2y3Q1vFSpLC/hx58bmXxHw8hTrKPZY5qK0M+QykirbI3KCsKciUBShM+fkRlmFFIg1lDLpJPX8slRHkVXFyOB4qnhdItciCO0DCPfMKOQAunoujy1CZwyokfOz5lrV1BRYHbWSBEzCimwcPWWhNvXbt3VJq0YFV0qraNYOssrT1BRupiK7/EzMsSMQgqs+uSg0o+l98b/4Lk2aYXcp5ANJVuMetoP7Dobbilqo/CXt9exfXd6o4Lc6vp87VMoTWFReF/Jz8trGIGlaI3Cik07+PJv5vP1h95M63i3uj5PbUJasYOuOH6gB5K4I09MWM6JzO6uLCvxWRIjXyjaeQo79x4A4MNPdmaUTzKdn6+L7JS48De03uXrUw9N+Twt5ink56UKNBdP6s/Fk/r7LYaRRxRtSyFTCr0DORSjpTC+v7sgc7GuzNmNvWLuG8v22NBUw/AP34yCiKxwlt9cICLznLTOIvKMiCx1/nfyS75ska8dzSUiDKjvkHCfVFT3j84ayYpbT27+feHEfmlKZhiGl/jdUjhOVUepaqPz+1rgWVUdBDzr/A40yVwembqn/CJWR7PrESwuDKEfLS1rfxhGcvw2Cq05HbjX+X4vcEauTrxl514279jbIm3v/ibWbGk79wAK3/89qndtTs5TlHMHDCPA+NnRrMBfRUSBX6vqHKCbqq51tq8DurU+SERmAbMA+vTpkzVhRv37M23Svvnwmzy+4CPe/fdpzSt6RQsPsGvv/qzJECQmDqxLXpePp9Bd6PlYRjWtGeNpHGMYRnz8NApHq+oaEekKPCMii6M3qqo6BoNW6XOAOQCNjY2e6YTNO/by7OINAOzed6CNUYhotXtfWumVCIFnYH1V7A0p3JVstROswWEY2cE395GqrnH+bwAeA8YA60WkAcD5v8Ev+Ubf/Azbd4dbAT/407t+iRFoxg9Ib8nLaAVuytwwgoUvRkFEOohIdeQ7MBVYBDwJnO/sdj7whB/ytebpt9a1STO3RXwSzc14+bopvH7jiYlnepuhMAzf8Mt91A14zOlkLAUeUNU/i8hrwEMichGwEjjbJ/mMDNi570Dcbd07tgOgX114uGvP2sqcyGQYhjt8MQqq+j7QZvkqVf0YmJJ7iYxsMrR7De+s3ZZwn/PG92VIQw3joibEeT6iK0YLpKqiaCf1G0ZM7I1Ik0Ifkgqk7SO7/+KxPLd4A1//Q/y4UqGQtDAIfvDUFUfTtSb5YkKGUUwUrVHItIOz0MNcxMLtNevUoZyROZrnkEkU2sN7dsyiJIZRGARt8lrOSEWXxFI8v3x+eRalKUTSV9bp2OtEBuvLx1hAOMNwS9EahVTYsfcAmz7d47cYGdO+9VyLVsw+bkDWzhUk99qgrtV+i2AYeUPRGoVU3UfzVmz2RpAA8dkjWkYyzUSvB8gmGIaRAkXbp5Aq+5ua/BbBUy49ZgD9481QznOi3X//c14j3Wra+SiNYQQbMwou2X8g/+u+8RpHg7tVce30ITmVJRbRnfe/vWgs/3bXKzH3+8XM0fTp3D7l/AXhhGFtwmm14YnZE1mZp9FtDSNTzCi4ZO+Bwm4pZJtM+hREhN6d409qO3Vkj4PnSf80cRnZuzZno6cMI2gUbZ9CqmQy9DGI1LYv8zT/dIbshjIYJ2yrtRlGdij6lsLiddv52TPvJd3v+08vTrpP0Ileu8BrFZqODb1+xlAqSkOcOrKBdVt3Z1+oNHnksvG8+kHhDzQwDLCWAgC3P7s06T5bd+3LgSTecubons3fow1EPAWeSesonUM7dyjnljOHU1GaeOhsrjnykM5cdmz2husaRpAxo1BEXDllUPP3u85vTLBn5uRqxvc3pw1hypCuTD0sfgdyYTn+DMNbzCgUCVdNGUR99cE4P6P7dEp6zE/PHsWkQXVeihUXt9FTe9ZWcteXjqJ9eXJPaKGv3fDNaUNoPKSTb/fMKAzMKBhxa9JHHtKJ31w0tvl3Kp25mfbLl5bYo5kqA7tW8fBlE+hgkV+NDLA3z8hKp/NFR/fLQi6GYfhN0RqFRWu2+i2CJwzu5s+s5G+fMqzF7yCN4D1mcD0lIeH88X39FsUwAk/RGoVvPLzQbxE84a9fOyZmeiId7VZ/z5qcn9FGu9W0Y/n3ZzC8l4XKNoxk5NwoiEhvEXleRN4RkbdF5Con/SYRWSMiC5zPDK9kaGoKUDXWA35+zsFF7UIR31CCqrvboaeTB9ez4taTXe1bjOtNGEYh4EdLYT/wdVUdBowDZotIxPfwc1Ud5Xye9kyAPDMK3zvtMH702RGu94/W8VdNGRx3v1LHYnx+7CHNaccP6Ur3LASMC5L7yAgePWsrOWZwvd9iGDHI+TAFVV0LrHW+bxeRd4GeiY/KLk15qLGmDO3a4veZo3vy2BtrXB8fq8T11RWs3bqb6Yd3b067+0tHpStiC3qnEbDOyB4je9eyOsBB/f557fF+i2DEwdc+BRHpC4wGIuEwLxeRhSJyt4jEHEgvIrNEZJ6IzNu4cWNa5823lkIsfn7OqLjbom1e67H5Q7pXU1bi/YD9zh3KXbua4jHKgtKlzROzJzL/2yf6LYaRh/hmFESkCngE+KqqbgN+BQwARhFuSfw01nGqOkdVG1W1sb4+vebngTwzCplMuhrbrzMA4/p3AeBPV03ivf+YDsBZR4YX1amp9DY4Xro89pUJfPADz7qWDMOIgS9GQUTKCBuE+1X1UQBVXa+qB1S1CbgTGOPV+YPS0dy7cyU3njzU1b6pSBwxIqeO7MHY/l1YfPM0Jg6sc7ZJc9yjq08czOKbp1GVwWSnMY7R8YKIrB2SLCNqGEb2yHmfgoQ10l3Au6r6s6j0Bqe/AeBMYJFXMgTFfVQWCiVUyEcPrOPFZZtSzjfiPop0JLcri61URSTuNrfcf/FY9u73dq0Jc4MYRu7wYz78ROCLwFsissBJux6YKSKjCFeKVwBf9kqAXHU0j+nbmVdXfJJwn0SuofLScEMukfeoojTEnjhKORehfspKQpR5HJIiU8NlGIZ7/Bh99CKx9ZVnQ1Bbk6s+hYqyJMpSEg/dbJ5jINJiv6ENNQDMv/EENu/cywk/m9viOC9K16+ugwe5GoYRNIpyRnNQOprj1eTLnZr34T3DM3AP6dyedo6BOaJPLQ9fOh6ALlUViaODZqmp8NJ1x/N/Vxzd/HuEzQw2jIKlKMMp5sooRNfuZ47pzYOvrmLSoDpeWHqwnyCW+yiSdt74vpwwtFuzcfjb1ZPp1al9C3dKrJL0qA1PPhvcrTrjMgA0dGwZxvqBS8axafseV8e+cv2UjJbZNAwjtxRnS8HDPoX/veAo2scYLdO5QzkAndqXN68VMLJXLd2jFO4PPzucZ742mX0Hwn0EFaWhZoMAMLBrtSv/+oQBdTz6lQnMmuRNrKKqilL6unQndatp12IdB8Mwgo21FLLI2Y29OO7QrhzRp1ObUUPRduiZqydz30sruXBiP8pLQzxw8ViaFCYO7IKIEBGvojS5zY4Xt+gIF4voGIZhtMaMQhYZ0y88QSziLamrKm+zjwi0Ly/l0mMOrvk7YWDLlbL6dmnPio93ulpoJrKecV1VBY/PnpCu6IZhGIAZhaxS22pm8Bmje/L4go8AUlqM/qFLx7N8ww5X+9ZXV3DPBUcxuk8nOgZ0ZrJhGPlDURoFr+YptA5aB/Dmd6fyr2Wb2LXvAOBuQFDX6nZ0rXYfqfTYQ9ue1zAMIx2KsqO5rir9js+K0hAPXDKWJy+f2GZbJHyERI226VhZxvThDc2dz7Xt27qUDMMwgkJRGoUetZXcf/HY5DvGYOaYPkwYUMeIXrWUhBLX+6PbI1OHdeemU4fxrWlD0jqvYRhGLihK9xHAxIF13HvhGM6/+9WE+10/Ywiq0KTwwz8vbjEk9C9fncRba7YyqGs1H0bFrq+qCO9TFjpoc0Mh4UsTbXF7wzCCTdEaBQgv6D73G8dxw+NvcdHR/ehZW8mrKz7hhsfCsfh+8rmRzeGl9+5vYte+A1x6zMGx/wO7VjOwa3iCWPR8glvOGM7Q7jVMGNAlh6UxDMPIHHG7Pm8QaWxs1Hnz5mU933+8t5Htu/dxyogeWc/bMAzDb0Rkvqo2xtpW1C2FeNjasYZhFCtF2dFsGIZhxMaMgmEYhtGMGQXDMAyjGTMKhmEYRjOBMwoiMk1ElojIMhG51m95DMMwiolAGQURKQF+CUwHhhFet3mYv1IZhmEUD4EyCsAYYJmqvq+qe4HfAaf7LJNhGEbREDSj0BNYFfV7tZPWjIjMEpF5IjJv48aNORXOMAyj0Mm7yWuqOgeYAyAiG0VkZQbZ1QGbku4VfAqlHGBlCSqFUpZCKQdkVpZD4m0ImlFYA/SO+t3LSYuJqmY09VhE5sWb6p1PFEo5wMoSVAqlLIVSDvCuLEFzH70GDBKRfiJSDpwLPOmzTIZhGEVDoFoKqrpfRC4H/gKUAHer6ts+i2UYhlE0BMooAKjq08DTOTrdnBydx2sKpRxgZQkqhVKWQikHeFSWvA6dbRiGYWSXoPUpGIZhGD5iRsEwDMNopiiNQj7GVxKRFSLylogsEJF5TlpnEXlGRJY6/zs56SIitzvlWygiR/gs+90iskFEFkWlpSy7iJzv7L9URM4PSDluEpE1zn1ZICIzorZd55RjiYicFJXu+/MnIr1F5HkReUdE3haRq5z0vLovCcqRd/dFRNqJyKsi8qZTlu856f1E5BVHrt87IzMRkQrn9zJne99kZXSFqhbVh/CopuVAf6AceBMY5rdcLuReAdS1SvsRcK3z/Vrgh873GcCfAAHGAa/4LPtk4AhgUbqyA52B953/nZzvnQJQjpuAa2LsO8x5tiqAfs4zVxKU5w9oAI5wvlcD7zky59V9SVCOvLsvzrWtcr6XAa841/oh4Fwn/Q7gMuf7V4A7nO/nAr9PVEa3chRjS6GQ4iudDtzrfL8XOCMq/T4N8zJQKyINPsgHgKrOBT5plZyq7CcBz6jqJ6q6GXgGmOa58FHEKUc8Tgd+p6p7VPUDYBnhZy8Qz5+qrlXV153v24F3CYeUyav7kqAc8QjsfXGu7afOzzLno8DxwMNOeut7ErlXDwNTRESIX0ZXFKNRSBpfKaAo8FcRmS8is5y0bqq61vm+DujmfM+HMqYqe5DLdLnjUrk74m4hj8rhuB1GE66Z5u19aVUOyMP7IiIlIrIA2EDYwC4Htqjq/hhyNcvsbN8KdCHDshSjUchXjlbVIwiHFZ8tIpOjN2q43ZiX44vzWXbgV8AAYBSwFvipr9KkiIhUAY8AX1XVbdHb8um+xChHXt4XVT2gqqMIh/gZAwzJtQzFaBRSiq8UFFR1jfN/A/AY4QdmfcQt5Pzf4OyeD2VMVfZAlklV1zsvchNwJweb6YEvh4iUEVak96vqo05y3t2XWOXI5/sCoKpbgOeB8YRddZGJxtFyNcvsbO8IfEyGZSlGo5B38ZVEpIOIVEe+A1OBRYTljoz2OB94wvn+JHCeM2JkHLA1yiUQFFKV/S/AVBHp5LgCpjppvtKqr+ZMwvcFwuU41xkh0g8YBLxKQJ4/x/d8F/Cuqv4salNe3Zd45cjH+yIi9SJS63yvBE4k3EfyPHCWs1vrexK5V2cBzzmtu3hldEcue9eD8iE8kuI9wv66G/yWx4W8/QmPJngTeDsiM2H/4bPAUuBvQGc9OIrhl0753gIafZb/QcJN+H2E/ZsXpSM7cCHhTrNlwAUBKcdvHDkXOi9jQ9T+NzjlWAJMD9LzBxxN2DW0EFjgfGbk231JUI68uy/ACOANR+ZFwHec9P6Elfoy4A9AhZPezvm9zNneP1kZ3XwszIVhGIbRTDG6jwzDMIw4mFEwDMMwmjGjYBiGYTRjRsEwDMNoxoyCYRiG0YwZBcOIQkQOREXWXJAsWqaIXCoi52XhvCtEpC7TfAwjU2xIqmFEISKfqmqVD+ddQXjs/6Zcn9sworGWgmG4wKnJ/0jCa1q8KiIDnfSbROQa5/uVEo7rv1BEfuekdRaRx520l0VkhJPeRUT+6sTN/x/Ck8Mi5/o35xwLROTXIlLiQ5GNIsWMgmG0pLKV++icqG1bVXU48F/Af8Y49lpgtKqOAC510r4HvOGkXQ/c56R/F3hRVQ8jHMuqD4CIDAXOASZqODDaAeAL2SygYSSiNPkuhlFU7HKUcSwejPr/8xjbFwL3i8jjwONO2tHAZwFU9TmnhVBDeMGezzjpfxSRzc7+U4AjgdfCYX2o5GBQOsPwHDMKhuEejfM9wsmElf2pwA0iMjyNcwhwr6pel8axhpEx5j4yDPecE/X/pegNIhICeqvq88C3CIcxrgJewHH/iMixwCYNx/ufC3zeSZ9OeClLCAejO0tEujrbOovIId4VyTBaYi0Fw2hJpbPyVYQ/q2pkWGonEVkI7AFmtjquBPitiHQkXNu/XVW3iMhNwN3OcTs5GOr4e8CDIvI28C/gQwBVfUdEbiS8yl6IcETW2cDKLJfTMGJiQ1INwwU2ZNQoFsx9ZBiGYTRjLQXDMAyjGWspGIZhGM2YUTAMwzCaMaNgGIZhNGNGwTAMw2jGjIJhGIbRzP8Do9u/judwoa4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "class Policy(Model):\n",
    "    def __init__(self, action_size):\n",
    "        super().__init__()\n",
    "        self.l1 = L.Linear(128)\n",
    "        self.l2 = L.Linear(action_size)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.l1(x))\n",
    "        x = F.softmax(self.l2(x))\n",
    "        return x\n",
    "\n",
    "\n",
    "class Agent:\n",
    "    def __init__(self):\n",
    "        self.gamma = 0.98\n",
    "        self.lr = 0.0002\n",
    "        self.action_size = 2\n",
    "\n",
    "        self.memory = []\n",
    "        self.pi = Policy(self.action_size)\n",
    "        self.optimizer = optimizers.Adam(self.lr)\n",
    "        self.optimizer.setup(self.pi)\n",
    "\n",
    "    def get_action(self, state):\n",
    "        state = state[np.newaxis, :]\n",
    "        probs = self.pi(state)\n",
    "        probs = probs[0]\n",
    "        action = np.random.choice(len(probs), p=probs.data)\n",
    "        return action, probs[action]\n",
    "\n",
    "    def add(self, reward, prob):\n",
    "        data = (reward, prob)\n",
    "        self.memory.append(data)\n",
    "\n",
    "    def update(self):\n",
    "        self.pi.cleargrads()\n",
    "\n",
    "        G, loss = 0, 0\n",
    "        for reward, prob in reversed(self.memory):\n",
    "            G = reward + self.gamma * G\n",
    "            loss += -F.log(prob) * G\n",
    "\n",
    "        loss.backward()\n",
    "        self.optimizer.update()\n",
    "        self.memory = []\n",
    "\n",
    "\n",
    "episodes = 3000\n",
    "env = gym.make('CartPole-v0')\n",
    "agent = Agent()\n",
    "reward_history = []\n",
    "\n",
    "for episode in range(episodes):\n",
    "    state = env.reset()\n",
    "    done = False\n",
    "    sum_reward = 0\n",
    "\n",
    "    while not done:\n",
    "        action, prob = agent.get_action(state)\n",
    "        next_state, reward, done, info = env.step(action)\n",
    "\n",
    "        agent.add(reward, prob)\n",
    "        state = next_state\n",
    "        sum_reward += reward\n",
    "\n",
    "    agent.update()\n",
    "\n",
    "    reward_history.append(sum_reward)\n",
    "    if episode % 100 == 0:\n",
    "        print(\"episode :{}, total reward : {:.1f}\".format(episode, sum_reward))\n",
    "\n",
    "\n",
    "# plot\n",
    "plot_total_reward(reward_history)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3372fddf",
   "metadata": {},
   "source": [
    "## ch09/actor_critic.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "59a22521",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode :0, total reward : 41.0\n",
      "episode :100, total reward : 12.0\n",
      "episode :200, total reward : 10.0\n",
      "episode :300, total reward : 15.0\n",
      "episode :400, total reward : 67.0\n",
      "episode :500, total reward : 54.0\n",
      "episode :600, total reward : 166.0\n",
      "episode :700, total reward : 200.0\n",
      "episode :800, total reward : 151.0\n",
      "episode :900, total reward : 200.0\n",
      "episode :1000, total reward : 200.0\n"
     ]
    }
   ],
   "source": [
    "class PolicyNet(Model):\n",
    "    def __init__(self, action_size=2):\n",
    "        super().__init__()\n",
    "        self.l1 = L.Linear(128)\n",
    "        self.l2 = L.Linear(action_size)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.l1(x))\n",
    "        x = self.l2(x)\n",
    "        x = F.softmax(x)\n",
    "        return x\n",
    "\n",
    "\n",
    "class ValueNet(Model):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.l1 = L.Linear(128)\n",
    "        self.l2 = L.Linear(1)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.l1(x))\n",
    "        x = self.l2(x)\n",
    "        return x\n",
    "\n",
    "\n",
    "class Agent:\n",
    "    def __init__(self):\n",
    "        self.gamma = 0.98\n",
    "        self.lr_pi = 0.0002\n",
    "        self.lr_v = 0.0005\n",
    "        self.action_size = 2\n",
    "\n",
    "        self.pi = PolicyNet()\n",
    "        self.v = ValueNet()\n",
    "        self.optimizer_pi = optimizers.Adam(self.lr_pi).setup(self.pi)\n",
    "        self.optimizer_v = optimizers.Adam(self.lr_v).setup(self.v)\n",
    "\n",
    "    def get_action(self, state):\n",
    "        state = state[np.newaxis, :]  # add batch axis\n",
    "        probs = self.pi(state)\n",
    "        probs = probs[0]\n",
    "        action = np.random.choice(len(probs), p=probs.data)\n",
    "        return action, probs[action]\n",
    "\n",
    "    def update(self, state, action_prob, reward, next_state, done):\n",
    "        state = state[np.newaxis, :]  # add batch axis\n",
    "        next_state = next_state[np.newaxis, :]\n",
    "\n",
    "        # ========== (1) Update V network ===========\n",
    "        target = reward + self.gamma * self.v(next_state) * (1 - done)\n",
    "        target.unchain()\n",
    "        v = self.v(state)\n",
    "        loss_v = F.mean_squared_error(v, target)\n",
    "\n",
    "        # ========== (2) Update pi network ===========\n",
    "        delta = target - v\n",
    "        delta.unchain()\n",
    "        loss_pi = -F.log(action_prob) * delta\n",
    "\n",
    "        self.v.cleargrads()\n",
    "        self.pi.cleargrads()\n",
    "        loss_v.backward()\n",
    "        loss_pi.backward()\n",
    "        self.optimizer_v.update()\n",
    "        self.optimizer_pi.update()\n",
    "\n",
    "\n",
    "episodes = 3000\n",
    "env = gym.make('CartPole-v0')\n",
    "agent = Agent()\n",
    "reward_history = []\n",
    "\n",
    "for episode in range(episodes):\n",
    "    state = env.reset()\n",
    "    done = False\n",
    "    total_reward = 0\n",
    "\n",
    "    while not done:\n",
    "        action, prob = agent.get_action(state)\n",
    "        next_state, reward, done, info = env.step(action)\n",
    "\n",
    "        agent.update(state, prob, reward, next_state, done)\n",
    "\n",
    "        state = next_state\n",
    "        total_reward += reward\n",
    "\n",
    "    reward_history.append(total_reward)\n",
    "    if episode % 100 == 0:\n",
    "        print(\"episode :{}, total reward : {:.1f}\".format(episode, total_reward))\n",
    "\n",
    "\n",
    "# plot\n",
    "plot_total_reward(reward_history)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
